From 52a7720e47f638fa057359766bc8e4178b9bf801 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Tue, 10 Jun 2025 09:00:33 +0200 Subject: [PATCH 01/21] added copy/paste commands to package.json --- package.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/package.json b/package.json index d774780db..19c3288c4 100644 --- a/package.json +++ b/package.json @@ -440,6 +440,18 @@ "category": "DocumentDB", "command": "vscode-documentdb.command.containerView.open", "title": "Open Collection" + }, + { + "//": "Copy Collection", + "category": "DocumentDB", + "command": "vscode-documentdb.command.copyCollection", + "title": "Copy Collection…" + }, + { + "//": "Paste Collection", + "category": "DocumentDB", + "command": "vscode-documentdb.command.pasteCollection", + "title": "Paste Collection…" } ], "submenus": [ @@ -672,6 +684,18 @@ "command": "vscode-documentdb.command.refresh", "when": "view =~ /discoveryView/ && viewItem =~ /\\benableRefreshCommand\\b/i", "group": "zheLastGroup@1" + }, + { + "//": "[Collection] Copy Collection", + "command": "vscode-documentdb.command.copyCollection", + "when": "view =~ /connectionsView|discoveryView/ && viewItem =~ /treeitem[.]collection(?![a-z.\\/])/i && viewItem =~ /experience[.](mongocluster|mongodb)/i", + "group": "A@2" + }, + { + "//": "[Collection] Paste Collection", + "command": "vscode-documentdb.command.pasteCollection", + "when": "view =~ /connectionsView|discoveryView/ && viewItem =~ /treeitem[.]collection(?![a-z.\\/])/i && viewItem =~ /experience[.](mongocluster|mongodb)/i", + "group": "A@2" } ], "explorer/context": [], From 33f7d401cd83a9f97b140153f88ced78446c2de4 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Tue, 10 Jun 2025 09:13:53 +0200 Subject: [PATCH 02/21] feat: basic copy+paste UX for experimenting --- l10n/bundle.l10n.json | 4 +++ src/commands/copyCollection/copyCollection.ts | 25 ++++++++++++++ .../pasteCollection/pasteCollection.ts | 34 +++++++++++++++++++ src/documentdb/ClustersExtension.ts | 5 +++ src/extensionVariables.ts | 4 +++ 5 files changed, 72 insertions(+) create mode 100644 src/commands/copyCollection/copyCollection.ts create mode 100644 src/commands/pasteCollection/pasteCollection.ts diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index 7921215f5..044cc13bd 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -69,6 +69,7 @@ "Click here to retry": "Click here to retry", "Click here to update credentials": "Click here to update credentials", "Click to view resource": "Click to view resource", + "Collection \"{0}\" from database \"{1}\" has been marked for copy.": "Collection \"{0}\" from database \"{1}\" has been marked for copy.", "Collection name cannot begin with the system. prefix (Reserved for internal use).": "Collection name cannot begin with the system. prefix (Reserved for internal use).", "Collection name cannot contain .system.": "Collection name cannot contain .system.", "Collection name cannot contain the $.": "Collection name cannot contain the $.", @@ -275,6 +276,7 @@ "No": "No", "No Azure subscription found for this tree item.": "No Azure subscription found for this tree item.", "No Azure VMs found with tag \"{tagName}\" in subscription \"{subscriptionName}\".": "No Azure VMs found with tag \"{tagName}\" in subscription \"{subscriptionName}\".", + "No collection has been marked for copy. Please use Copy Collection first.": "No collection has been marked for copy. Please use Copy Collection first.", "No collection selected.": "No collection selected.", "No commands found in this document.": "No commands found in this document.", "No Connectivity": "No Connectivity", @@ -350,6 +352,7 @@ "Skip for now": "Skip for now", "Small breadcrumb example with buttons": "Small breadcrumb example with buttons", "Some items could not be displayed": "Some items could not be displayed", + "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "Specified character lengths should be 1 character or greater.": "Specified character lengths should be 1 character or greater.", "Started executable: \"{command}\". Connecting to host…": "Started executable: \"{command}\". Connecting to host…", "Starting executable: \"{command}\"": "Starting executable: \"{command}\"", @@ -363,6 +366,7 @@ "Tag can only contain alphanumeric characters, underscores, periods, and hyphens.": "Tag can only contain alphanumeric characters, underscores, periods, and hyphens.", "Tag cannot be empty.": "Tag cannot be empty.", "Tag cannot be longer than 256 characters.": "Tag cannot be longer than 256 characters.", + "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "The \"{databaseId}\" database has been deleted.": "The \"{databaseId}\" database has been deleted.", "The \"{name}\" database has been created.": "The \"{name}\" database has been created.", "The \"{newCollectionName}\" collection has been created.": "The \"{newCollectionName}\" collection has been created.", diff --git a/src/commands/copyCollection/copyCollection.ts b/src/commands/copyCollection/copyCollection.ts new file mode 100644 index 000000000..b9f1f72b8 --- /dev/null +++ b/src/commands/copyCollection/copyCollection.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type IActionContext } from '@microsoft/vscode-azext-utils'; +import * as vscode from 'vscode'; +import { ext } from '../../extensionVariables'; +import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; + +export async function copyCollection(_context: IActionContext, node: CollectionItem): Promise { + if (!node) { + throw new Error(vscode.l10n.t('No node selected.')); + } + // Store the node in extension variables + ext.copiedCollectionNode = node; + + // Show confirmation message + const collectionName = node.collectionInfo.name; + const databaseName = node.databaseInfo.name; + + void vscode.window.showInformationMessage( + vscode.l10n.t('Collection "{0}" from database "{1}" has been marked for copy.', collectionName, databaseName), + ); +} diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts new file mode 100644 index 000000000..de1d3b23c --- /dev/null +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type IActionContext } from '@microsoft/vscode-azext-utils'; +import * as vscode from 'vscode'; +import { ext } from '../../extensionVariables'; +import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; + +export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { + const sourceNode = ext.copiedCollectionNode as CollectionItem | undefined; + if (!sourceNode) { + void vscode.window.showWarningMessage( + vscode.l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), + ); + return; + } + + const sourceInfo = vscode.l10n.t( + 'Source: Collection "{0}" from database "{1}", connectionId: {2}', + sourceNode.collectionInfo.name, + sourceNode.databaseInfo.name, + sourceNode.cluster.id, + ); + const targetInfo = vscode.l10n.t( + 'Target: Collection "{0}" from database "{1}", connectionId: {2}', + targetNode.collectionInfo.name, + targetNode.databaseInfo.name, + targetNode.cluster.id, + ); + + void vscode.window.showInformationMessage(`${sourceInfo}\n${targetInfo}`); +} diff --git a/src/documentdb/ClustersExtension.ts b/src/documentdb/ClustersExtension.ts index a08adafc4..298e54d1f 100644 --- a/src/documentdb/ClustersExtension.ts +++ b/src/documentdb/ClustersExtension.ts @@ -19,6 +19,7 @@ import { AzExtResourceType } from '@microsoft/vscode-azureresources-api'; import * as vscode from 'vscode'; import { addConnectionFromRegistry } from '../commands/addConnectionFromRegistry/addConnectionFromRegistry'; import { addDiscoveryRegistry } from '../commands/addDiscoveryRegistry/addDiscoveryRegistry'; +import { copyCollection } from '../commands/copyCollection/copyCollection'; import { copyAzureConnectionString } from '../commands/copyConnectionString/copyConnectionString'; import { createCollection } from '../commands/createCollection/createCollection'; import { createAzureDatabase } from '../commands/createDatabase/createDatabase'; @@ -34,6 +35,7 @@ import { newConnection } from '../commands/newConnection/newConnection'; import { newLocalConnection } from '../commands/newLocalConnection/newLocalConnection'; import { openCollectionView, openCollectionViewInternal } from '../commands/openCollectionView/openCollectionView'; import { openDocumentView } from '../commands/openDocument/openDocument'; +import { pasteCollection } from '../commands/pasteCollection/pasteCollection'; import { refreshTreeElement } from '../commands/refreshTreeElement/refreshTreeElement'; import { refreshView } from '../commands/refreshView/refreshView'; import { removeConnection } from '../commands/removeConnection/removeConnection'; @@ -197,6 +199,9 @@ export class ClustersExtension implements vscode.Disposable { renameConnection, ); + registerCommandWithTreeNodeUnwrapping('vscode-documentdb.command.copyCollection', copyCollection); + registerCommandWithTreeNodeUnwrapping('vscode-documentdb.command.pasteCollection', pasteCollection); + // using registerCommand instead of vscode.commands.registerCommand for better telemetry: // https://github.com/microsoft/vscode-azuretools/tree/main/utils#telemetry-and-error-handling diff --git a/src/extensionVariables.ts b/src/extensionVariables.ts index a7aecca12..6de9eab4a 100644 --- a/src/extensionVariables.ts +++ b/src/extensionVariables.ts @@ -11,6 +11,7 @@ import { type MongoDBLanguageClient } from './documentdb/scrapbook/languageClien import { type MongoVCoreBranchDataProvider } from './tree/azure-resources-view/documentdb/mongo-vcore/MongoVCoreBranchDataProvider'; import { type ConnectionsBranchDataProvider } from './tree/connections-view/ConnectionsBranchDataProvider'; import { type DiscoveryBranchDataProvider } from './tree/discovery-view/DiscoveryBranchDataProvider'; +import { type CollectionItem } from './tree/documentdb/CollectionItem'; import { type AccountsItem } from './tree/workspace-view/documentdb/AccountsItem'; import { type ClustersWorkspaceBranchDataProvider } from './tree/workspace-view/documentdb/ClustersWorkbenchBranchDataProvider'; @@ -26,6 +27,9 @@ export namespace ext { export let fileSystem: DatabasesFileSystem; export let mongoLanguageClient: MongoDBLanguageClient; + // TODO: TN imporove this: This is a temporary solution to get going. + export let copiedCollectionNode: CollectionItem | undefined; + // Since the Azure Resources extension did not update API interface, but added a new interface with activity // we have to use the new interface AzureResourcesExtensionApiWithActivity instead of AzureResourcesExtensionApi export let rgApiV2: AzureResourcesExtensionApiWithActivity; From aca6916678bf233e793d99c0f8002b94b41d1a7d Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Tue, 10 Jun 2025 09:15:55 +0200 Subject: [PATCH 03/21] fix: updated eslint config to exclude the `api/dist` folder --- .eslintignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintignore b/.eslintignore index a4a973227..2f606ee30 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,4 +1,5 @@ /dist +/api/dist /out /node_modules **/__mocks__/** From eab759347df491dfc51eef494ceafc57d518c541 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Tue, 10 Jun 2025 09:59:49 +0200 Subject: [PATCH 04/21] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/commands/pasteCollection/pasteCollection.ts | 2 +- src/extensionVariables.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index de1d3b23c..12d2dcacb 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -9,7 +9,7 @@ import { ext } from '../../extensionVariables'; import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { - const sourceNode = ext.copiedCollectionNode as CollectionItem | undefined; + const sourceNode = ext.copiedCollectionNode; if (!sourceNode) { void vscode.window.showWarningMessage( vscode.l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), diff --git a/src/extensionVariables.ts b/src/extensionVariables.ts index 6de9eab4a..0fd1a66d4 100644 --- a/src/extensionVariables.ts +++ b/src/extensionVariables.ts @@ -27,7 +27,7 @@ export namespace ext { export let fileSystem: DatabasesFileSystem; export let mongoLanguageClient: MongoDBLanguageClient; - // TODO: TN imporove this: This is a temporary solution to get going. + // TODO: TN improve this: This is a temporary solution to get going. export let copiedCollectionNode: CollectionItem | undefined; // Since the Azure Resources extension did not update API interface, but added a new interface with activity From 52eec8d1fa63a58caec439705816857fb350915f Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Mon, 16 Jun 2025 15:25:23 +0000 Subject: [PATCH 05/21] draft copy paste task --- .../pasteCollection/pasteCollection.ts | 160 ++++++++- src/documentdb/ClustersClient.ts | 16 + src/documentdb/DocumentProvider.ts | 138 ++++++++ src/services/tasks/CopyPasteCollectionTask.ts | 310 ++++++++++++++++++ src/utils/copyPasteUtils.ts | 131 ++++++++ 5 files changed, 745 insertions(+), 10 deletions(-) create mode 100644 src/documentdb/DocumentProvider.ts create mode 100644 src/services/tasks/CopyPasteCollectionTask.ts create mode 100644 src/utils/copyPasteUtils.ts diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 12d2dcacb..05e779d57 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -4,31 +4,171 @@ *--------------------------------------------------------------------------------------------*/ import { type IActionContext } from '@microsoft/vscode-azext-utils'; +import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; +import { MongoDocumentReader, MongoDocumentWriter } from '../../documentdb/DocumentProvider'; import { ext } from '../../extensionVariables'; -import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; +import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; +import { TaskService, TaskState } from '../../services/taskService'; +import { CollectionItem } from '../../tree/documentdb/CollectionItem'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../utils/copyPasteUtils'; -export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { +export async function pasteCollection(context: IActionContext, targetNode: CollectionItem): Promise { const sourceNode = ext.copiedCollectionNode; if (!sourceNode) { void vscode.window.showWarningMessage( - vscode.l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), + l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), ); return; } - const sourceInfo = vscode.l10n.t( - 'Source: Collection "{0}" from database "{1}", connectionId: {2}', + if (!targetNode) { + throw new Error(l10n.t('No target node selected.')); + } + + // Check type of sourceNode or targetNode + // Currently we only support CollectionItem types + // Later we need to check if they are supported types that with document reader and writer implementations + if (!(sourceNode instanceof CollectionItem) || !(targetNode instanceof CollectionItem)) { + void vscode.window.showWarningMessage(vscode.l10n.t('Invalid source or target node type.')); + return; + } + + // Confirm the copy operation with the user + const sourceInfo = l10n.t( + 'Source: Collection "{0}" from database "{1}"', sourceNode.collectionInfo.name, sourceNode.databaseInfo.name, - sourceNode.cluster.id, ); - const targetInfo = vscode.l10n.t( - 'Target: Collection "{0}" from database "{1}", connectionId: {2}', + const targetInfo = l10n.t( + 'Target: Collection "{0}" from database "{1}"', targetNode.collectionInfo.name, targetNode.databaseInfo.name, - targetNode.cluster.id, ); - void vscode.window.showInformationMessage(`${sourceInfo}\n${targetInfo}`); + // Confirm the copy operation with the user + const confirmMessage = l10n.t( + 'Copy "{0}"\nto "{1}"?\nThis will add all documents from the source collection to the target collection.', + sourceInfo, + targetInfo, + ); + + const confirmation = await vscode.window.showWarningMessage(confirmMessage, { modal: true }, l10n.t('Copy')); + + if (confirmation !== l10n.t('Copy')) { + return; + } + + try { + // Create copy-paste configuration + const config: CopyPasteConfig = { + source: { + connectionId: sourceNode.cluster.id, + databaseName: sourceNode.databaseInfo.name, + collectionName: sourceNode.collectionInfo.name, + }, + target: { + connectionId: targetNode.cluster.id, + databaseName: targetNode.databaseInfo.name, + collectionName: targetNode.collectionInfo.name, + }, + // Currently we only support aborting on conflict + onConflict: ConflictResolutionStrategy.Abort, + }; + + // Create task with documentDB document providers + // Need to check reader and writer implementations before creating the task + // For now, we only support MongoDB collections + const reader = new MongoDocumentReader(); + const writer = new MongoDocumentWriter(); + const task = new CopyPasteCollectionTask(config, reader, writer); + + // Get total number of documents in the source collection + const totalDocuments = await reader.countDocuments( + config.source.connectionId, + config.source.databaseName, + config.source.collectionName, + ); + + // Register task with the task service + TaskService.registerTask(task); + + // Show progress notification + await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: l10n.t('Initializing copy task...'), + cancellable: true, + }, + async (progress, token) => { + progress.report({ increment: 0, message: l10n.t('Copying documents…') }); + // Handle cancellation + token.onCancellationRequested(() => { + void task.stop(); + }); + + // Start the task + await task.start(); + + // Monitor progress + let lastProgress = 0; + while ( + task.getStatus().state === TaskState.Running || + task.getStatus().state === TaskState.Initializing + ) { + const status = task.getStatus(); + const currentProgress = status.progress || 0; + + if (currentProgress > lastProgress) { + progress.report({ + increment: ((currentProgress - lastProgress) / totalDocuments) * 100, + message: status.message, + }); + lastProgress = currentProgress; + } + + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + // Final progress update + const finalStatus = task.getStatus(); + if (finalStatus.state === TaskState.Completed) { + progress.report({ + increment: 100 - (lastProgress / totalDocuments) * 100, + message: finalStatus.message, + }); + } + }, + ); + + // Check final status and show result + const finalStatus = task.getStatus(); + if (finalStatus.state === TaskState.Completed) { + void vscode.window.showInformationMessage( + l10n.t('Collection copied successfully: {0}', finalStatus.message || ''), + ); + } else if (finalStatus.state === TaskState.Failed) { + const errorToThrow = + finalStatus.error instanceof Error ? finalStatus.error : new Error('Copy operation failed'); + throw errorToThrow; + } else if (finalStatus.state === TaskState.Stopped) { + void vscode.window.showInformationMessage(l10n.t('Copy operation was cancelled.')); + } + } catch (error) { + context.telemetry.properties.error = 'true'; + const errorMessage = error instanceof Error ? error.message : String(error); + void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); + throw error; + } finally { + // Clean up - remove the task from the service after completion + try { + const task = TaskService.listTasks().find((t) => t.type === 'copy-paste-collection'); + if (task) { + await TaskService.deleteTask(task.id); + } + } catch (cleanupError) { + // Log cleanup error but don't throw + console.warn('Failed to clean up copy-paste task:', cleanupError); + } + } } diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index 09efafbf8..9b3f0ec70 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -286,6 +286,22 @@ export class ClustersClient { return documents; } + async countDocuments(databaseName: string, collectionName: string, findQuery: string = '{}'): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + if (findQuery === undefined || findQuery.trim().length === 0) { + findQuery = '{}'; + } + const findQueryObj: Filter = toFilterQueryObj(findQuery); + const collection = this._mongoClient.db(databaseName).collection(collectionName); + + const count = await collection.countDocuments(findQueryObj, { + // Use a read preference of 'primary' to ensure we get the most up-to-date + // count, especially important for sharded clusters. + readPreference: 'primary', + }); + return count; + } + async *streamDocuments( databaseName: string, collectionName: string, diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts new file mode 100644 index 000000000..0808922b1 --- /dev/null +++ b/src/documentdb/DocumentProvider.ts @@ -0,0 +1,138 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type Document, type WithId } from 'mongodb'; +import { ClustersClient } from '../documentdb/ClustersClient'; +import { + type BulkWriteResult, + type DocumentDetails, + type DocumentReader, + type DocumentWriter, + type DocumentWriterOptions, +} from '../utils/copyPasteUtils'; + +/** + * MongoDB-specific implementation of DocumentReader + */ +export class MongoDocumentReader implements DocumentReader { + /** + * Stream documents from MongoDB collection + */ + public async *streamDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + ): AsyncIterable { + const client = await ClustersClient.getClient(connectionId); + + // Use ClustersClient's streamDocuments method + const docStream = client.streamDocuments(databaseName, collectionName, new AbortController().signal); + + for await (const document of docStream) { + yield { + id: (document as WithId)._id, + documentContent: document, + }; + } + } + + /** + * Count documents in MongoDB collection + */ + public async countDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + filter: string = '{}', + ): Promise { + const client = await ClustersClient.getClient(connectionId); + + return await client.countDocuments(databaseName, collectionName, filter); + } +} + +/** + * MongoDB-specific implementation of DocumentWriter + */ +export class MongoDocumentWriter implements DocumentWriter { + /** + * Write documents to MongoDB collection using bulk operations + */ + public async writeDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + documents: DocumentDetails[], + _options?: DocumentWriterOptions, + ): Promise { + const client = await ClustersClient.getClient(connectionId); + + // Convert DocumentDetails to MongoDB documents + const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); + + try { + const result = await client.insertDocuments(databaseName, collectionName, mongoDocuments); + + return { + insertedCount: result.insertedCount, + // todo: update later + errors: [], + }; + } catch (error: unknown) { + // Handle MongoDB bulk write errors + const errors: Array<{ documentId?: unknown; error: Error }> = []; + + if (error && typeof error === 'object' && 'writeErrors' in error) { + const writeErrors = (error as { writeErrors: unknown[] }).writeErrors; + for (const writeError of writeErrors) { + if (writeError && typeof writeError === 'object' && 'index' in writeError) { + const docIndex = writeError.index as number; + const documentId = docIndex < documents.length ? documents[docIndex].id : undefined; + const errorMessage = + 'errmsg' in writeError ? (writeError.errmsg as string) : 'Unknown write error'; + errors.push({ + documentId, + error: new Error(errorMessage), + }); + } + } + } else { + errors.push({ + error: error instanceof Error ? error : new Error(String(error)), + }); + } + + const insertedCount = + error && typeof error === 'object' && 'result' in error + ? ((error as { result?: { insertedCount?: number } }).result?.insertedCount ?? 0) + : 0; + + return { + insertedCount, + errors, + }; + } + } + + /** + * Ensure MongoDB collection exists + */ + public async ensureCollectionExists( + connectionId: string, + databaseName: string, + collectionName: string, + ): Promise { + const client = await ClustersClient.getClient(connectionId); + + // Check if collection exists by trying to list collections + const collections = await client.listCollections(databaseName); + const collectionExists = collections.some((col) => col.name === collectionName); + + if (!collectionExists) { + // Create the collection by running createCollection + await client.createCollection(databaseName, collectionName); + } + } +} diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts new file mode 100644 index 000000000..64e42fa91 --- /dev/null +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -0,0 +1,310 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { v4 as uuidv4 } from 'uuid'; +import { l10n } from 'vscode'; +import { ext } from '../../extensionVariables'; +import { + type BulkWriteResult, + ConflictResolutionStrategy, + type CopyPasteConfig, + type DocumentDetails, + type DocumentReader, + type DocumentWriter, +} from '../../utils/copyPasteUtils'; +import { BufferErrorCode, createMongoDbBuffer } from '../../utils/documentBuffer'; +import { type Task, TaskState, type TaskStatus } from '../taskService'; + +/** + * Implementation of a copy-paste collection task using buffer-based streaming + */ +export class CopyPasteCollectionTask implements Task { + public readonly id: string; + public readonly type: string = 'copy-paste-collection'; + public readonly name: string; + private totalDocuments: number = 0; + + private status: TaskStatus; + private isRunning: boolean = false; + private shouldStop: boolean = false; + private documentBuffer = createMongoDbBuffer(); + private copiedDocumentCount: number = 0; + + constructor( + private readonly config: CopyPasteConfig, + private readonly reader: DocumentReader, + private readonly writer: DocumentWriter, + ) { + this.id = uuidv4(); + this.name = `Copy collection ${config.source.collectionName} to ${config.target.collectionName}`; + this.status = { + state: TaskState.Pending, + progress: 0, + message: 'Task created', + }; + void reader + .countDocuments(config.source.connectionId, config.source.databaseName, config.source.collectionName) + .then((count) => { + this.totalDocuments = count || 0; + }); + } + + /** + * Get the current status of the task + */ + public getStatus(): TaskStatus { + return { ...this.status }; + } + + /** + * Start the copy-paste operation + */ + public async start(): Promise { + if (this.isRunning) { + throw new Error('Task is already running'); + } + + if (this.status.state !== TaskState.Pending) { + throw new Error(`Cannot start task in state: ${this.status.state}`); + } + + this.isRunning = true; + this.shouldStop = false; + + try { + await this.executeTask(); + } catch (error) { + this.updateStatus({ + state: TaskState.Failed, + error: error instanceof Error ? error : new Error(String(error)), + message: `Task failed: ${error instanceof Error ? error.message : String(error)}`, + }); + throw error; + } finally { + this.isRunning = false; + } + } + + /** + * Stop the task gracefully + */ + public async stop(): Promise { + if (!this.isRunning) { + return; + } + + this.shouldStop = true; + this.updateStatus({ + state: TaskState.Stopping, + message: 'Stopping task...', + }); + + // Wait for the task to acknowledge the stop request + while (this.isRunning && this.status.state === TaskState.Stopping) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + /** + * Clean up resources + */ + public async delete(): Promise { + if (this.isRunning) { + await this.stop(); + } + // Additional cleanup if needed + } + + /** + * Execute the copy-paste operation with buffer-based streaming + */ + private async executeTask(): Promise { + try { + // Get total document number + this.updateStatus({ + state: TaskState.Initializing, + progress: 0, + message: 'Counting source documents...', + }); + + if (this.shouldStop) { + this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during initialization' }); + return; + } + + // Ensure target collection exists + this.updateStatus({ + state: TaskState.Initializing, + progress: 5, + message: 'Ensuring target collection exists...', + }); + + await this.writer.ensureCollectionExists( + this.config.target.connectionId, + this.config.target.databaseName, + this.config.target.collectionName, + ); + + if (this.shouldStop) { + this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during setup' }); + return; + } + + // Start the streaming copy + this.updateStatus({ + state: TaskState.Running, + progress: 10, + message: 'Starting document copy...', + }); + + await this.streamDocuments(); + + if (this.shouldStop) { + this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during copy' }); + return; + } + + // Complete + this.updateStatus({ + state: TaskState.Completed, + progress: 100, + message: `Successfully copied ${this.copiedDocumentCount} documents`, + }); + } catch (error) { + if (this.config.onConflict === ConflictResolutionStrategy.Abort) { + throw error; + } + // For future conflict resolution strategies, handle them here + throw error; + } + } + + /** + * Stream documents using buffer-based approach + */ + private async streamDocuments(): Promise { + const documents = this.reader.streamDocuments( + this.config.source.connectionId, + this.config.source.databaseName, + this.config.source.collectionName, + ); + + // Read documents and buffer them + for await (const document of documents) { + if (this.shouldStop) { + break; + } + + // Try to add document to buffer + const insertResult = this.documentBuffer.insert(document); + if (insertResult.success) { + // Successfully inserted into buffer, continue to next document + continue; + } + + // Handle insert failures + if (insertResult.errorCode === BufferErrorCode.DocumentTooLarge) { + // Document is too large for buffer, handle immediately + await this.writeDocuments([document]); + continue; + } else if (insertResult.errorCode === BufferErrorCode.BufferFull) { + // Buffer is full, flush first + if (this.documentBuffer.getStats().documentCount > 0) { + await this.flushBuffer(); + } + // Insert again after flush + // We checked for DocumentTooLarge above, so we can safely retry + const retryInsertResult = this.documentBuffer.insert(document); + if (!retryInsertResult.success) { + // If still fails, log the error and continue + ext.outputChannel.appendLog( + l10n.t( + 'Failed to insert document with id {0} into buffer: {1}', + document.id as string, + String(retryInsertResult.errorCode), + ), + ); + } + continue; + } else { + ext.outputChannel.appendLog( + l10n.t( + 'Failed to insert document with id {0} into buffer: {1}', + document.id as string, + String(insertResult.errorCode), + ), + ); + continue; + } + } + + // Flush any remaining documents in the buffer + if (this.documentBuffer.getStats().documentCount > 0) { + await this.flushBuffer(); + } + } + + /** + * Flush the document buffer to the target collection + */ + private async flushBuffer(): Promise { + const documents = this.documentBuffer.flush(); + if (documents.length > 0) { + await this.writeDocuments(documents); + } + } + + /** + * Write documents to the target collection with error handling + */ + private async writeDocuments(documents: DocumentDetails[]): Promise { + try { + const result: BulkWriteResult = await this.writer.writeDocuments( + this.config.target.connectionId, + this.config.target.databaseName, + this.config.target.collectionName, + documents, + ); + this.copiedDocumentCount += result.insertedCount; + this.updateProgress(this.copiedDocumentCount); + + // Handle write errors based on conflict resolution strategy + if (result.errors.length > 0 && this.config.onConflict === ConflictResolutionStrategy.Abort) { + const firstError = result.errors[0]; + throw new Error( + `Write operation failed: ${firstError.error.message}. Document ID: ${firstError.documentId}`, + ); + } + } catch (error) { + if (this.config.onConflict === ConflictResolutionStrategy.Abort) { + throw error; + } + // For future conflict resolution strategies, handle them here + throw error; + } + } + + /** + * Update task progress + */ + private updateProgress(current: number): void { + const progress = Math.min(Math.round((current / this.totalDocuments) * 90) + 10, 100); // Reserve 10% for setup + this.updateStatus({ + state: TaskState.Running, + progress, + message: `Copied ${current} of ${this.totalDocuments} documents`, + }); + } + + /** + * Update task status + */ + private updateStatus(updates: Partial): void { + this.status = { + ...this.status, + ...updates, + }; + } +} diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts new file mode 100644 index 000000000..68ed5e8de --- /dev/null +++ b/src/utils/copyPasteUtils.ts @@ -0,0 +1,131 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Conflict resolution strategies for copy-paste operations + */ +export enum ConflictResolutionStrategy { + /** + * Abort the operation if any conflict is encountered + */ + Abort = 'abort', + // Future options: Overwrite = 'overwrite', Skip = 'skip' +} + +/** + * Configuration for copy-paste operations + */ +export interface CopyPasteConfig { + /** + * Source collection information + */ + source: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Target collection information + */ + target: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Conflict resolution strategy + */ + onConflict: ConflictResolutionStrategy; + + /** + * Optional reference to a connection manager or client object. + * For now, this is typed as `unknown` to allow flexibility. + * Specific task implementations (e.g., for MongoDB) will cast this to their + * required client/connection type. + */ + connectionManager?: unknown; +} + +/** + * Represents a single document for copy-paste operations + */ +export interface DocumentDetails { + /** + * The document's unique identifier (e.g., _id in MongoDB) + */ + id: unknown; + + /** + * The document content as opaque data + * For MongoDB, this would typically be a BSON document + */ + documentContent: unknown; +} + +/** + * Interface for reading documents from a source + */ +export interface DocumentReader { + /** + * Streams documents from the source collection + */ + streamDocuments(connectionId: string, databaseName: string, collectionName: string): AsyncIterable; + + /** + * Counts documents in the source collection for progress calculation + */ + countDocuments(connectionId: string, databaseName: string, collectionName: string): Promise; +} + +/** + * Options for document writer operations + */ +export interface DocumentWriterOptions { + /** + * Batch size for bulk operations + */ + batchSize?: number; +} + +/** + * Result of bulk write operations + */ +export interface BulkWriteResult { + /** + * Number of documents successfully inserted + */ + insertedCount: number; + + /** + * Array of errors that occurred during the operation + */ + errors: Array<{ + documentId?: unknown; + error: Error; + }>; +} + +/** + * Interface for writing documents to a target + */ +export interface DocumentWriter { + /** + * Writes documents in bulk to the target collection + */ + writeDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + documents: DocumentDetails[], + options?: DocumentWriterOptions, + ): Promise; + + /** + * Ensures the target collection exists + */ + ensureCollectionExists(connectionId: string, databaseName: string, collectionName: string): Promise; +} From 3a737fb890a4f2689abd9bbce94cd64295b64e9c Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Thu, 19 Jun 2025 08:59:02 +0000 Subject: [PATCH 06/21] Revert "draft copy paste task" This reverts commit 52eec8d1fa63a58caec439705816857fb350915f. --- .../pasteCollection/pasteCollection.ts | 160 +-------- src/documentdb/ClustersClient.ts | 16 - src/documentdb/DocumentProvider.ts | 138 -------- src/services/tasks/CopyPasteCollectionTask.ts | 310 ------------------ src/utils/copyPasteUtils.ts | 131 -------- 5 files changed, 10 insertions(+), 745 deletions(-) delete mode 100644 src/documentdb/DocumentProvider.ts delete mode 100644 src/services/tasks/CopyPasteCollectionTask.ts delete mode 100644 src/utils/copyPasteUtils.ts diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 05e779d57..12d2dcacb 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -4,171 +4,31 @@ *--------------------------------------------------------------------------------------------*/ import { type IActionContext } from '@microsoft/vscode-azext-utils'; -import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; -import { MongoDocumentReader, MongoDocumentWriter } from '../../documentdb/DocumentProvider'; import { ext } from '../../extensionVariables'; -import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; -import { TaskService, TaskState } from '../../services/taskService'; -import { CollectionItem } from '../../tree/documentdb/CollectionItem'; -import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../utils/copyPasteUtils'; +import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; -export async function pasteCollection(context: IActionContext, targetNode: CollectionItem): Promise { +export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { const sourceNode = ext.copiedCollectionNode; if (!sourceNode) { void vscode.window.showWarningMessage( - l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), + vscode.l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), ); return; } - if (!targetNode) { - throw new Error(l10n.t('No target node selected.')); - } - - // Check type of sourceNode or targetNode - // Currently we only support CollectionItem types - // Later we need to check if they are supported types that with document reader and writer implementations - if (!(sourceNode instanceof CollectionItem) || !(targetNode instanceof CollectionItem)) { - void vscode.window.showWarningMessage(vscode.l10n.t('Invalid source or target node type.')); - return; - } - - // Confirm the copy operation with the user - const sourceInfo = l10n.t( - 'Source: Collection "{0}" from database "{1}"', + const sourceInfo = vscode.l10n.t( + 'Source: Collection "{0}" from database "{1}", connectionId: {2}', sourceNode.collectionInfo.name, sourceNode.databaseInfo.name, + sourceNode.cluster.id, ); - const targetInfo = l10n.t( - 'Target: Collection "{0}" from database "{1}"', + const targetInfo = vscode.l10n.t( + 'Target: Collection "{0}" from database "{1}", connectionId: {2}', targetNode.collectionInfo.name, targetNode.databaseInfo.name, + targetNode.cluster.id, ); - // Confirm the copy operation with the user - const confirmMessage = l10n.t( - 'Copy "{0}"\nto "{1}"?\nThis will add all documents from the source collection to the target collection.', - sourceInfo, - targetInfo, - ); - - const confirmation = await vscode.window.showWarningMessage(confirmMessage, { modal: true }, l10n.t('Copy')); - - if (confirmation !== l10n.t('Copy')) { - return; - } - - try { - // Create copy-paste configuration - const config: CopyPasteConfig = { - source: { - connectionId: sourceNode.cluster.id, - databaseName: sourceNode.databaseInfo.name, - collectionName: sourceNode.collectionInfo.name, - }, - target: { - connectionId: targetNode.cluster.id, - databaseName: targetNode.databaseInfo.name, - collectionName: targetNode.collectionInfo.name, - }, - // Currently we only support aborting on conflict - onConflict: ConflictResolutionStrategy.Abort, - }; - - // Create task with documentDB document providers - // Need to check reader and writer implementations before creating the task - // For now, we only support MongoDB collections - const reader = new MongoDocumentReader(); - const writer = new MongoDocumentWriter(); - const task = new CopyPasteCollectionTask(config, reader, writer); - - // Get total number of documents in the source collection - const totalDocuments = await reader.countDocuments( - config.source.connectionId, - config.source.databaseName, - config.source.collectionName, - ); - - // Register task with the task service - TaskService.registerTask(task); - - // Show progress notification - await vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: l10n.t('Initializing copy task...'), - cancellable: true, - }, - async (progress, token) => { - progress.report({ increment: 0, message: l10n.t('Copying documents…') }); - // Handle cancellation - token.onCancellationRequested(() => { - void task.stop(); - }); - - // Start the task - await task.start(); - - // Monitor progress - let lastProgress = 0; - while ( - task.getStatus().state === TaskState.Running || - task.getStatus().state === TaskState.Initializing - ) { - const status = task.getStatus(); - const currentProgress = status.progress || 0; - - if (currentProgress > lastProgress) { - progress.report({ - increment: ((currentProgress - lastProgress) / totalDocuments) * 100, - message: status.message, - }); - lastProgress = currentProgress; - } - - await new Promise((resolve) => setTimeout(resolve, 50)); - } - - // Final progress update - const finalStatus = task.getStatus(); - if (finalStatus.state === TaskState.Completed) { - progress.report({ - increment: 100 - (lastProgress / totalDocuments) * 100, - message: finalStatus.message, - }); - } - }, - ); - - // Check final status and show result - const finalStatus = task.getStatus(); - if (finalStatus.state === TaskState.Completed) { - void vscode.window.showInformationMessage( - l10n.t('Collection copied successfully: {0}', finalStatus.message || ''), - ); - } else if (finalStatus.state === TaskState.Failed) { - const errorToThrow = - finalStatus.error instanceof Error ? finalStatus.error : new Error('Copy operation failed'); - throw errorToThrow; - } else if (finalStatus.state === TaskState.Stopped) { - void vscode.window.showInformationMessage(l10n.t('Copy operation was cancelled.')); - } - } catch (error) { - context.telemetry.properties.error = 'true'; - const errorMessage = error instanceof Error ? error.message : String(error); - void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); - throw error; - } finally { - // Clean up - remove the task from the service after completion - try { - const task = TaskService.listTasks().find((t) => t.type === 'copy-paste-collection'); - if (task) { - await TaskService.deleteTask(task.id); - } - } catch (cleanupError) { - // Log cleanup error but don't throw - console.warn('Failed to clean up copy-paste task:', cleanupError); - } - } + void vscode.window.showInformationMessage(`${sourceInfo}\n${targetInfo}`); } diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index 9b3f0ec70..09efafbf8 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -286,22 +286,6 @@ export class ClustersClient { return documents; } - async countDocuments(databaseName: string, collectionName: string, findQuery: string = '{}'): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - if (findQuery === undefined || findQuery.trim().length === 0) { - findQuery = '{}'; - } - const findQueryObj: Filter = toFilterQueryObj(findQuery); - const collection = this._mongoClient.db(databaseName).collection(collectionName); - - const count = await collection.countDocuments(findQueryObj, { - // Use a read preference of 'primary' to ensure we get the most up-to-date - // count, especially important for sharded clusters. - readPreference: 'primary', - }); - return count; - } - async *streamDocuments( databaseName: string, collectionName: string, diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts deleted file mode 100644 index 0808922b1..000000000 --- a/src/documentdb/DocumentProvider.ts +++ /dev/null @@ -1,138 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type Document, type WithId } from 'mongodb'; -import { ClustersClient } from '../documentdb/ClustersClient'; -import { - type BulkWriteResult, - type DocumentDetails, - type DocumentReader, - type DocumentWriter, - type DocumentWriterOptions, -} from '../utils/copyPasteUtils'; - -/** - * MongoDB-specific implementation of DocumentReader - */ -export class MongoDocumentReader implements DocumentReader { - /** - * Stream documents from MongoDB collection - */ - public async *streamDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - ): AsyncIterable { - const client = await ClustersClient.getClient(connectionId); - - // Use ClustersClient's streamDocuments method - const docStream = client.streamDocuments(databaseName, collectionName, new AbortController().signal); - - for await (const document of docStream) { - yield { - id: (document as WithId)._id, - documentContent: document, - }; - } - } - - /** - * Count documents in MongoDB collection - */ - public async countDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - filter: string = '{}', - ): Promise { - const client = await ClustersClient.getClient(connectionId); - - return await client.countDocuments(databaseName, collectionName, filter); - } -} - -/** - * MongoDB-specific implementation of DocumentWriter - */ -export class MongoDocumentWriter implements DocumentWriter { - /** - * Write documents to MongoDB collection using bulk operations - */ - public async writeDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - documents: DocumentDetails[], - _options?: DocumentWriterOptions, - ): Promise { - const client = await ClustersClient.getClient(connectionId); - - // Convert DocumentDetails to MongoDB documents - const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); - - try { - const result = await client.insertDocuments(databaseName, collectionName, mongoDocuments); - - return { - insertedCount: result.insertedCount, - // todo: update later - errors: [], - }; - } catch (error: unknown) { - // Handle MongoDB bulk write errors - const errors: Array<{ documentId?: unknown; error: Error }> = []; - - if (error && typeof error === 'object' && 'writeErrors' in error) { - const writeErrors = (error as { writeErrors: unknown[] }).writeErrors; - for (const writeError of writeErrors) { - if (writeError && typeof writeError === 'object' && 'index' in writeError) { - const docIndex = writeError.index as number; - const documentId = docIndex < documents.length ? documents[docIndex].id : undefined; - const errorMessage = - 'errmsg' in writeError ? (writeError.errmsg as string) : 'Unknown write error'; - errors.push({ - documentId, - error: new Error(errorMessage), - }); - } - } - } else { - errors.push({ - error: error instanceof Error ? error : new Error(String(error)), - }); - } - - const insertedCount = - error && typeof error === 'object' && 'result' in error - ? ((error as { result?: { insertedCount?: number } }).result?.insertedCount ?? 0) - : 0; - - return { - insertedCount, - errors, - }; - } - } - - /** - * Ensure MongoDB collection exists - */ - public async ensureCollectionExists( - connectionId: string, - databaseName: string, - collectionName: string, - ): Promise { - const client = await ClustersClient.getClient(connectionId); - - // Check if collection exists by trying to list collections - const collections = await client.listCollections(databaseName); - const collectionExists = collections.some((col) => col.name === collectionName); - - if (!collectionExists) { - // Create the collection by running createCollection - await client.createCollection(databaseName, collectionName); - } - } -} diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts deleted file mode 100644 index 64e42fa91..000000000 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ /dev/null @@ -1,310 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { v4 as uuidv4 } from 'uuid'; -import { l10n } from 'vscode'; -import { ext } from '../../extensionVariables'; -import { - type BulkWriteResult, - ConflictResolutionStrategy, - type CopyPasteConfig, - type DocumentDetails, - type DocumentReader, - type DocumentWriter, -} from '../../utils/copyPasteUtils'; -import { BufferErrorCode, createMongoDbBuffer } from '../../utils/documentBuffer'; -import { type Task, TaskState, type TaskStatus } from '../taskService'; - -/** - * Implementation of a copy-paste collection task using buffer-based streaming - */ -export class CopyPasteCollectionTask implements Task { - public readonly id: string; - public readonly type: string = 'copy-paste-collection'; - public readonly name: string; - private totalDocuments: number = 0; - - private status: TaskStatus; - private isRunning: boolean = false; - private shouldStop: boolean = false; - private documentBuffer = createMongoDbBuffer(); - private copiedDocumentCount: number = 0; - - constructor( - private readonly config: CopyPasteConfig, - private readonly reader: DocumentReader, - private readonly writer: DocumentWriter, - ) { - this.id = uuidv4(); - this.name = `Copy collection ${config.source.collectionName} to ${config.target.collectionName}`; - this.status = { - state: TaskState.Pending, - progress: 0, - message: 'Task created', - }; - void reader - .countDocuments(config.source.connectionId, config.source.databaseName, config.source.collectionName) - .then((count) => { - this.totalDocuments = count || 0; - }); - } - - /** - * Get the current status of the task - */ - public getStatus(): TaskStatus { - return { ...this.status }; - } - - /** - * Start the copy-paste operation - */ - public async start(): Promise { - if (this.isRunning) { - throw new Error('Task is already running'); - } - - if (this.status.state !== TaskState.Pending) { - throw new Error(`Cannot start task in state: ${this.status.state}`); - } - - this.isRunning = true; - this.shouldStop = false; - - try { - await this.executeTask(); - } catch (error) { - this.updateStatus({ - state: TaskState.Failed, - error: error instanceof Error ? error : new Error(String(error)), - message: `Task failed: ${error instanceof Error ? error.message : String(error)}`, - }); - throw error; - } finally { - this.isRunning = false; - } - } - - /** - * Stop the task gracefully - */ - public async stop(): Promise { - if (!this.isRunning) { - return; - } - - this.shouldStop = true; - this.updateStatus({ - state: TaskState.Stopping, - message: 'Stopping task...', - }); - - // Wait for the task to acknowledge the stop request - while (this.isRunning && this.status.state === TaskState.Stopping) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - /** - * Clean up resources - */ - public async delete(): Promise { - if (this.isRunning) { - await this.stop(); - } - // Additional cleanup if needed - } - - /** - * Execute the copy-paste operation with buffer-based streaming - */ - private async executeTask(): Promise { - try { - // Get total document number - this.updateStatus({ - state: TaskState.Initializing, - progress: 0, - message: 'Counting source documents...', - }); - - if (this.shouldStop) { - this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during initialization' }); - return; - } - - // Ensure target collection exists - this.updateStatus({ - state: TaskState.Initializing, - progress: 5, - message: 'Ensuring target collection exists...', - }); - - await this.writer.ensureCollectionExists( - this.config.target.connectionId, - this.config.target.databaseName, - this.config.target.collectionName, - ); - - if (this.shouldStop) { - this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during setup' }); - return; - } - - // Start the streaming copy - this.updateStatus({ - state: TaskState.Running, - progress: 10, - message: 'Starting document copy...', - }); - - await this.streamDocuments(); - - if (this.shouldStop) { - this.updateStatus({ state: TaskState.Stopped, message: 'Task stopped during copy' }); - return; - } - - // Complete - this.updateStatus({ - state: TaskState.Completed, - progress: 100, - message: `Successfully copied ${this.copiedDocumentCount} documents`, - }); - } catch (error) { - if (this.config.onConflict === ConflictResolutionStrategy.Abort) { - throw error; - } - // For future conflict resolution strategies, handle them here - throw error; - } - } - - /** - * Stream documents using buffer-based approach - */ - private async streamDocuments(): Promise { - const documents = this.reader.streamDocuments( - this.config.source.connectionId, - this.config.source.databaseName, - this.config.source.collectionName, - ); - - // Read documents and buffer them - for await (const document of documents) { - if (this.shouldStop) { - break; - } - - // Try to add document to buffer - const insertResult = this.documentBuffer.insert(document); - if (insertResult.success) { - // Successfully inserted into buffer, continue to next document - continue; - } - - // Handle insert failures - if (insertResult.errorCode === BufferErrorCode.DocumentTooLarge) { - // Document is too large for buffer, handle immediately - await this.writeDocuments([document]); - continue; - } else if (insertResult.errorCode === BufferErrorCode.BufferFull) { - // Buffer is full, flush first - if (this.documentBuffer.getStats().documentCount > 0) { - await this.flushBuffer(); - } - // Insert again after flush - // We checked for DocumentTooLarge above, so we can safely retry - const retryInsertResult = this.documentBuffer.insert(document); - if (!retryInsertResult.success) { - // If still fails, log the error and continue - ext.outputChannel.appendLog( - l10n.t( - 'Failed to insert document with id {0} into buffer: {1}', - document.id as string, - String(retryInsertResult.errorCode), - ), - ); - } - continue; - } else { - ext.outputChannel.appendLog( - l10n.t( - 'Failed to insert document with id {0} into buffer: {1}', - document.id as string, - String(insertResult.errorCode), - ), - ); - continue; - } - } - - // Flush any remaining documents in the buffer - if (this.documentBuffer.getStats().documentCount > 0) { - await this.flushBuffer(); - } - } - - /** - * Flush the document buffer to the target collection - */ - private async flushBuffer(): Promise { - const documents = this.documentBuffer.flush(); - if (documents.length > 0) { - await this.writeDocuments(documents); - } - } - - /** - * Write documents to the target collection with error handling - */ - private async writeDocuments(documents: DocumentDetails[]): Promise { - try { - const result: BulkWriteResult = await this.writer.writeDocuments( - this.config.target.connectionId, - this.config.target.databaseName, - this.config.target.collectionName, - documents, - ); - this.copiedDocumentCount += result.insertedCount; - this.updateProgress(this.copiedDocumentCount); - - // Handle write errors based on conflict resolution strategy - if (result.errors.length > 0 && this.config.onConflict === ConflictResolutionStrategy.Abort) { - const firstError = result.errors[0]; - throw new Error( - `Write operation failed: ${firstError.error.message}. Document ID: ${firstError.documentId}`, - ); - } - } catch (error) { - if (this.config.onConflict === ConflictResolutionStrategy.Abort) { - throw error; - } - // For future conflict resolution strategies, handle them here - throw error; - } - } - - /** - * Update task progress - */ - private updateProgress(current: number): void { - const progress = Math.min(Math.round((current / this.totalDocuments) * 90) + 10, 100); // Reserve 10% for setup - this.updateStatus({ - state: TaskState.Running, - progress, - message: `Copied ${current} of ${this.totalDocuments} documents`, - }); - } - - /** - * Update task status - */ - private updateStatus(updates: Partial): void { - this.status = { - ...this.status, - ...updates, - }; - } -} diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts deleted file mode 100644 index 68ed5e8de..000000000 --- a/src/utils/copyPasteUtils.ts +++ /dev/null @@ -1,131 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -/** - * Conflict resolution strategies for copy-paste operations - */ -export enum ConflictResolutionStrategy { - /** - * Abort the operation if any conflict is encountered - */ - Abort = 'abort', - // Future options: Overwrite = 'overwrite', Skip = 'skip' -} - -/** - * Configuration for copy-paste operations - */ -export interface CopyPasteConfig { - /** - * Source collection information - */ - source: { - connectionId: string; - databaseName: string; - collectionName: string; - }; - - /** - * Target collection information - */ - target: { - connectionId: string; - databaseName: string; - collectionName: string; - }; - - /** - * Conflict resolution strategy - */ - onConflict: ConflictResolutionStrategy; - - /** - * Optional reference to a connection manager or client object. - * For now, this is typed as `unknown` to allow flexibility. - * Specific task implementations (e.g., for MongoDB) will cast this to their - * required client/connection type. - */ - connectionManager?: unknown; -} - -/** - * Represents a single document for copy-paste operations - */ -export interface DocumentDetails { - /** - * The document's unique identifier (e.g., _id in MongoDB) - */ - id: unknown; - - /** - * The document content as opaque data - * For MongoDB, this would typically be a BSON document - */ - documentContent: unknown; -} - -/** - * Interface for reading documents from a source - */ -export interface DocumentReader { - /** - * Streams documents from the source collection - */ - streamDocuments(connectionId: string, databaseName: string, collectionName: string): AsyncIterable; - - /** - * Counts documents in the source collection for progress calculation - */ - countDocuments(connectionId: string, databaseName: string, collectionName: string): Promise; -} - -/** - * Options for document writer operations - */ -export interface DocumentWriterOptions { - /** - * Batch size for bulk operations - */ - batchSize?: number; -} - -/** - * Result of bulk write operations - */ -export interface BulkWriteResult { - /** - * Number of documents successfully inserted - */ - insertedCount: number; - - /** - * Array of errors that occurred during the operation - */ - errors: Array<{ - documentId?: unknown; - error: Error; - }>; -} - -/** - * Interface for writing documents to a target - */ -export interface DocumentWriter { - /** - * Writes documents in bulk to the target collection - */ - writeDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - documents: DocumentDetails[], - options?: DocumentWriterOptions, - ): Promise; - - /** - * Ensures the target collection exists - */ - ensureCollectionExists(connectionId: string, databaseName: string, collectionName: string): Promise; -} From 95ee9dad153d9a0fbabf236863554e2d4c16eab5 Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Mon, 30 Jun 2025 07:58:20 +0000 Subject: [PATCH 07/21] draft version for copy paste collection task --- .../pasteCollection/pasteCollection.ts | 112 +++++++- src/documentdb/ClustersClient.ts | 16 ++ src/documentdb/DocumentProvider.ts | 159 +++++++++++ src/services/tasks/CopyPasteCollectionTask.ts | 252 ++++++++++++++++++ src/utils/copyPasteUtils.ts | 155 +++++++++++ 5 files changed, 689 insertions(+), 5 deletions(-) create mode 100644 src/documentdb/DocumentProvider.ts create mode 100644 src/services/tasks/CopyPasteCollectionTask.ts create mode 100644 src/utils/copyPasteUtils.ts diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 12d2dcacb..e81b4d400 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -4,31 +4,133 @@ *--------------------------------------------------------------------------------------------*/ import { type IActionContext } from '@microsoft/vscode-azext-utils'; +import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; +import { MongoDocumentReader, MongoDocumentWriter } from '../../documentdb/DocumentProvider'; import { ext } from '../../extensionVariables'; -import { type CollectionItem } from '../../tree/documentdb/CollectionItem'; +import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; +import { TaskService, TaskState } from '../../services/taskService'; +import { CollectionItem } from '../../tree/documentdb/CollectionItem'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../utils/copyPasteUtils'; export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { const sourceNode = ext.copiedCollectionNode; if (!sourceNode) { void vscode.window.showWarningMessage( - vscode.l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), + l10n.t('No collection has been marked for copy. Please use Copy Collection first.'), ); return; } - const sourceInfo = vscode.l10n.t( + if (!targetNode) { + throw new Error(vscode.l10n.t('No target node selected.')); + } + + // Check type of sourceNode or targetNodeAdd commentMore actions + // Currently we only support CollectionItem types + // Later we need to check if they are supported types that with document reader and writer implementations + if (!(sourceNode instanceof CollectionItem) || !(targetNode instanceof CollectionItem)) { + void vscode.window.showWarningMessage(l10n.t('Invalid source or target node type.')); + return; + } + + const sourceInfo = l10n.t( 'Source: Collection "{0}" from database "{1}", connectionId: {2}', sourceNode.collectionInfo.name, sourceNode.databaseInfo.name, sourceNode.cluster.id, ); - const targetInfo = vscode.l10n.t( + const targetInfo = l10n.t( 'Target: Collection "{0}" from database "{1}", connectionId: {2}', targetNode.collectionInfo.name, targetNode.databaseInfo.name, targetNode.cluster.id, ); - void vscode.window.showInformationMessage(`${sourceInfo}\n${targetInfo}`); + // void vscode.window.showInformationMessage(`${sourceInfo}\n${targetInfo}`); + // Confirm the copy operation with the userAdd commentMore actions + const confirmMessage = l10n.t( + 'Copy "{0}"\nto "{1}"?\nThis will add all documents from the source collection to the target collection.', + sourceInfo, + targetInfo, + ); + + const confirmation = await vscode.window.showWarningMessage(confirmMessage, { modal: true }, l10n.t('Copy')); + + if (confirmation !== l10n.t('Copy')) { + return; + } + + try { + // Create copy-paste configuration + const config: CopyPasteConfig = { + source: { + connectionId: sourceNode.cluster.id, + databaseName: sourceNode.databaseInfo.name, + collectionName: sourceNode.collectionInfo.name, + }, + target: { + connectionId: targetNode.cluster.id, + databaseName: targetNode.databaseInfo.name, + collectionName: targetNode.collectionInfo.name, + }, + // Currently we only support aborting on conflict + onConflict: ConflictResolutionStrategy.Abort, + }; + + // Create task with documentDB document providers + // Need to check reader and writer implementations before creating the task + // For now, we only support MongoDB collections + const reader = new MongoDocumentReader(); + const writer = new MongoDocumentWriter(); + const task = new CopyPasteCollectionTask(config, reader, writer); + + // // Get total number of documents in the source collection + // const totalDocuments = await reader.countDocuments( + // config.source.connectionId, + // config.source.databaseName, + // config.source.collectionName, + // ); + + // Register task with the task service + TaskService.registerTask(task); + + // Start and monitor the task without showing a progress notification + await task.start(); + + // Wait for the task to complete + while (task.getStatus().state === TaskState.Running || task.getStatus().state === TaskState.Initializing) { + // Simple polling with a small delay + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + // Check final status and show result + const finalStatus = task.getStatus(); + if (finalStatus.state === TaskState.Completed) { + void vscode.window.showInformationMessage( + l10n.t('Collection copied successfully: {0}', finalStatus.message || ''), + ); + } else if (finalStatus.state === TaskState.Failed) { + const errorToThrow = + finalStatus.error instanceof Error ? finalStatus.error : new Error('Copy operation failed'); + throw errorToThrow; + } else if (finalStatus.state === TaskState.Stopped) { + void vscode.window.showInformationMessage(l10n.t('Copy operation was cancelled.')); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); + throw error; + } finally { + // Clean up - remove the task from the service after completion + try { + const task = TaskService.listTasks().find((t) => t.type === 'copy-paste-collection'); + if (task) { + await TaskService.deleteTask(task.id); + } + } catch (cleanupError) { + // Log cleanup error but don't throw + console.warn('Failed to clean up copy-paste task:', cleanupError); + } + } } diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index 09efafbf8..9b3f0ec70 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -286,6 +286,22 @@ export class ClustersClient { return documents; } + async countDocuments(databaseName: string, collectionName: string, findQuery: string = '{}'): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + if (findQuery === undefined || findQuery.trim().length === 0) { + findQuery = '{}'; + } + const findQueryObj: Filter = toFilterQueryObj(findQuery); + const collection = this._mongoClient.db(databaseName).collection(collectionName); + + const count = await collection.countDocuments(findQueryObj, { + // Use a read preference of 'primary' to ensure we get the most up-to-date + // count, especially important for sharded clusters. + readPreference: 'primary', + }); + return count; + } + async *streamDocuments( databaseName: string, collectionName: string, diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts new file mode 100644 index 000000000..0bc56a735 --- /dev/null +++ b/src/documentdb/DocumentProvider.ts @@ -0,0 +1,159 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type Document, type WithId } from 'mongodb'; +import { ClustersClient } from '../documentdb/ClustersClient'; +import { + type BulkWriteResult, + type DocumentDetails, + type DocumentReader, + type DocumentWriter, + type DocumentWriterOptions, +} from '../utils/copyPasteUtils'; + +/** + * MongoDB-specific implementation of DocumentReader. + */ +export class MongoDocumentReader implements DocumentReader { + /** + * Streams documents from a MongoDB collection. + * + * @param connectionId Connection identifier to get the MongoDB client + * @param databaseName Name of the database + * @param collectionName Name of the collection + * @returns AsyncIterable of document details + */ + async *streamDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + ): AsyncIterable { + const client = await ClustersClient.getClient(connectionId); + + const documentStream = client.streamDocuments(databaseName, collectionName, new AbortController().signal); + for await (const document of documentStream) { + yield { + id: (document as WithId)._id, + documentContent: document, + }; + } + } + + /** + * Counts the total number of documents in a MongoDB collection. + * + * @param connectionId Connection identifier to get the MongoDB client + * @param databaseName Name of the database + * @param collectionName Name of the collection, + * @param filter Optional filter to apply to the count operation (default is '{}') + * @returns Promise resolving to the document count + */ + async countDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + filter: string = '{}', + ): Promise { + const client = await ClustersClient.getClient(connectionId); + return await client.countDocuments(databaseName, collectionName, filter); + } +} + +/** + * MongoDB-specific implementation of DocumentWriter. + */ +export class MongoDocumentWriter implements DocumentWriter { + /** + * Writes documents to a MongoDB collection using bulk operations. + * + * @param connectionId Connection identifier to get the MongoDB client + * @param databaseName Name of the target database + * @param collectionName Name of the target collection + * @param documents Array of documents to write + * @param options Optional write options + * @returns Promise resolving to the bulk write result + */ + async writeDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + documents: DocumentDetails[], + _options?: DocumentWriterOptions, + ): Promise { + if (documents.length === 0) { + return { + insertedCount: 0, + errors: [], + }; + } + + try { + const client = await ClustersClient.getClient(connectionId); + + // Convert DocumentDetails to MongoDB documents + const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); + const result = await client.insertDocuments(databaseName, collectionName, mongoDocuments); + + return { + insertedCount: result.insertedCount, + errors: [], // ClustersClient.insertDocuments doesn't return detailed errors in the current implementation + }; + } catch (error: unknown) { + // Handle MongoDB bulk write errors + const errors: Array<{ documentId?: unknown; error: Error }> = []; + + if (error && typeof error === 'object' && 'writeErrors' in error) { + const writeErrors = (error as { writeErrors: unknown[] }).writeErrors; + for (const writeError of writeErrors) { + if (writeError && typeof writeError === 'object' && 'index' in writeError) { + const docIndex = writeError.index as number; + const documentId = docIndex < documents.length ? documents[docIndex].id : undefined; + const errorMessage = + 'errmsg' in writeError ? (writeError.errmsg as string) : 'Unknown write error'; + errors.push({ + documentId, + error: new Error(errorMessage), + }); + } + } + } else { + errors.push({ + error: error instanceof Error ? error : new Error(String(error)), + }); + } + + const insertedCount = + error && typeof error === 'object' && 'result' in error + ? ((error as { result?: { insertedCount?: number } }).result?.insertedCount ?? 0) + : 0; + + return { + insertedCount, + errors, + }; + } + } + + /** + * Ensures the target collection exists in MongoDB. + * + * @param connectionId Connection identifier to get the MongoDB client + * @param databaseName Name of the target database + * @param collectionName Name of the target collection + * @returns Promise that resolves when the collection is ready + */ + async ensureCollectionExists(connectionId: string, databaseName: string, collectionName: string): Promise { + const client = await ClustersClient.getClient(connectionId); + + // Check if collection exists by trying to list collections + const collections = await client.listCollections(databaseName); + const collectionExists = collections.some((col) => col.name === collectionName); + + if (!collectionExists) { + // Create the collection by running createCollection + await client.createCollection(databaseName, collectionName); + } + } +} diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts new file mode 100644 index 000000000..a422cc440 --- /dev/null +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -0,0 +1,252 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as vscode from 'vscode'; +import { + type CopyPasteConfig, + type DocumentDetails, + type DocumentReader, + type DocumentWriter, + ConflictResolutionStrategy, +} from '../../utils/copyPasteUtils'; +import { Task } from '../taskService'; + +/** + * Task for copying documents from a source collection to a target collection. + * + * This task implements a database-agnostic approach using DocumentReader and DocumentWriter + * interfaces to handle the actual data operations. It manages memory efficiently through + * a buffer-based streaming approach where documents are read and written in batches. + */ +export class CopyPasteCollectionTask extends Task { + public readonly type: string = 'copy-paste-collection'; + public readonly name: string; + + private readonly config: CopyPasteConfig; + private readonly documentReader: DocumentReader; + private readonly documentWriter: DocumentWriter; + private totalDocuments: number = 0; + private processedDocuments: number = 0; + + // Buffer configuration for memory management + private readonly bufferSize: number = 100; // Number of documents to buffer + private readonly maxBufferMemoryMB: number = 32; // Rough memory limit for buffer + + /** + * Creates a new CopyPasteCollectionTask instance. + * + * @param config Configuration for the copy-paste operation + * @param documentReader Reader implementation for the source database + * @param documentWriter Writer implementation for the target database + */ + constructor(config: CopyPasteConfig, documentReader: DocumentReader, documentWriter: DocumentWriter) { + super(); + this.config = config; + this.documentReader = documentReader; + this.documentWriter = documentWriter; + + // Generate a descriptive name for the task + this.name = vscode.l10n.t( + 'Copy collection "{0}" from "{1}" to "{2}"', + config.source.collectionName, + config.source.databaseName, + config.target.databaseName, + ); + } + + /** + * Initializes the task by counting documents and ensuring target collection exists. + * + * @param signal AbortSignal to check for cancellation + */ + protected async onInitialize(signal: AbortSignal): Promise { + // Count total documents for progress calculation + this.updateStatus(this.getStatus().state, vscode.l10n.t('Counting documents in source collection...'), 0); + + if (signal.aborted) { + return; + } + + try { + this.totalDocuments = await this.documentReader.countDocuments( + this.config.source.connectionId, + this.config.source.databaseName, + this.config.source.collectionName, + ); + } catch (error) { + throw new Error( + vscode.l10n.t( + 'Failed to count documents in source collection: {0}', + error instanceof Error ? error.message : String(error), + ), + ); + } + + if (signal.aborted) { + return; + } + + // Ensure target collection exists + this.updateStatus(this.getStatus().state, vscode.l10n.t('Ensuring target collection exists...'), 0); + + try { + await this.documentWriter.ensureCollectionExists( + this.config.target.connectionId, + this.config.target.databaseName, + this.config.target.collectionName, + ); + } catch (error) { + throw new Error( + vscode.l10n.t( + 'Failed to ensure target collection exists: {0}', + error instanceof Error ? error.message : String(error), + ), + ); + } + } + + /** + * Performs the main copy-paste operation using buffer-based streaming. + * + * @param signal AbortSignal to check for cancellation + */ + protected async doWork(signal: AbortSignal): Promise { + // Handle the case where there are no documents to copy + if (this.totalDocuments === 0) { + this.updateProgress(100, vscode.l10n.t('No documents to copy. Operation completed.')); + return; + } + + this.updateProgress(0, vscode.l10n.t('Starting document copy...')); + + const documentStream = this.documentReader.streamDocuments( + this.config.source.connectionId, + this.config.source.databaseName, + this.config.source.collectionName, + ); + + const buffer: DocumentDetails[] = []; + let bufferMemoryEstimate = 0; + + try { + for await (const document of documentStream) { + if (signal.aborted) { + // Cleanup any remaining buffer + if (buffer.length > 0) { + await this.flushBuffer(buffer); + } + return; + } + + // Add document to buffer + buffer.push(document); + bufferMemoryEstimate += this.estimateDocumentMemory(document); + + // Check if we need to flush the buffer + if (this.shouldFlushBuffer(buffer.length, bufferMemoryEstimate)) { + await this.flushBuffer(buffer); + buffer.length = 0; // Clear buffer + bufferMemoryEstimate = 0; + } + } + + // Flush any remaining documents in the buffer + if (buffer.length > 0) { + await this.flushBuffer(buffer); + } + + // Ensure we report 100% completion + this.updateProgress(100, vscode.l10n.t('Copy operation completed successfully')); + } catch (error) { + // For basic implementation, any error should abort the operation + if (this.config.onConflict === ConflictResolutionStrategy.Abort) { + throw new Error( + vscode.l10n.t('Copy operation failed: {0}', error instanceof Error ? error.message : String(error)), + ); + } + // Future: Handle other conflict resolution strategies + throw error; + } + } + + /** + * Flushes the document buffer by writing all documents to the target collection. + * + * @param buffer Array of documents to write + */ + private async flushBuffer(buffer: DocumentDetails[]): Promise { + if (buffer.length === 0) { + return; + } + + const result = await this.documentWriter.writeDocuments( + this.config.target.connectionId, + this.config.target.databaseName, + this.config.target.collectionName, + buffer, + { batchSize: buffer.length }, + ); + + // Update processed count + this.processedDocuments += result.insertedCount; + + // Check for errors in the write result + if (result.errors.length > 0) { + // For basic implementation with abort strategy, any error should fail the task + if (this.config.onConflict === ConflictResolutionStrategy.Abort) { + const firstError = result.errors[0]; + throw new Error(vscode.l10n.t('Write operation failed: {0}', firstError.error.message)); + } + // Future: Handle other conflict resolution strategies + } + + // Update progress + const progress = Math.min(100, (this.processedDocuments / this.totalDocuments) * 100); + this.updateProgress( + progress, + vscode.l10n.t('Copied {0} of {1} documents', this.processedDocuments, this.totalDocuments), + ); + } + + /** + * Determines whether the buffer should be flushed based on size and memory constraints. + * + * @param bufferCount Number of documents in the buffer + * @param memoryEstimate Estimated memory usage in bytes + * @returns True if the buffer should be flushed + */ + private shouldFlushBuffer(bufferCount: number, memoryEstimate: number): boolean { + // Flush if we've reached the document count limit + if (bufferCount >= this.bufferSize) { + return true; + } + + // Flush if we've exceeded the memory limit (converted to bytes) + const memoryLimitBytes = this.maxBufferMemoryMB * 1024 * 1024; + if (memoryEstimate >= memoryLimitBytes) { + return true; + } + + return false; + } + + /** + * Estimates the memory usage of a document in bytes. + * This is a rough estimate based on JSON serialization. + * + * @param document Document to estimate + * @returns Estimated memory usage in bytes + */ + private estimateDocumentMemory(document: DocumentDetails): number { + try { + // Rough estimate: JSON stringify the document content + const jsonString = JSON.stringify(document.documentContent); + return jsonString.length * 2; // Rough estimate for UTF-16 encoding + } catch { + // If we can't serialize, use a conservative estimate + return 1024; // 1KB default estimate + } + } +} diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts new file mode 100644 index 000000000..48e6dc9cc --- /dev/null +++ b/src/utils/copyPasteUtils.ts @@ -0,0 +1,155 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Enumeration of conflict resolution strategies for copy-paste operations + */ +export enum ConflictResolutionStrategy { + /** + * Abort the operation if any conflict or error occurs + */ + Abort = 'abort', + // Future options: Overwrite = 'overwrite', Skip = 'skip' +} + +/** + * Configuration for copy-paste operations + */ +export interface CopyPasteConfig { + /** + * Source collection information + */ + source: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Target collection information + */ + target: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Conflict resolution strategy + */ + onConflict: ConflictResolutionStrategy; + + /** + * Optional reference to a connection manager or client object. + * For now, this is typed as `unknown` to allow flexibility. + * Specific task implementations (e.g., for MongoDB) will cast this to their + * required client/connection type. + */ + connectionManager?: unknown; // e.g. could be cast to a MongoDB client instance +} + +/** + * Represents a single document in the copy-paste operation. + */ +export interface DocumentDetails { + /** + * The document's unique identifier (e.g., _id in MongoDB) + */ + id: unknown; + + /** + * The document content treated as opaque data by the core task logic. + * Specific readers/writers will know how to interpret/serialize this. + * For MongoDB, this would typically be a BSON document. + */ + documentContent: unknown; +} + +/** + * Interface for reading documents from a source collection + */ +export interface DocumentReader { + /** + * Streams documents from the source collection. + * + * @param connectionId Connection identifier for the source + * @param databaseName Name of the source database + * @param collectionName Name of the source collection + * @returns AsyncIterable of documents + */ + streamDocuments(connectionId: string, databaseName: string, collectionName: string): AsyncIterable; + + /** + * Counts documents in the source collection for progress calculation. + * + * @param connectionId Connection identifier for the source + * @param databaseName Name of the source database + * @param collectionName Name of the source collection + * @returns Promise resolving to the number of documents + */ + countDocuments(connectionId: string, databaseName: string, collectionName: string): Promise; +} + +/** + * Options for document writing operations. + */ +export interface DocumentWriterOptions { + /** + * Batch size for bulk write operations. + */ + batchSize?: number; +} + +/** + * Result of a bulk write operation. + */ +export interface BulkWriteResult { + /** + * Number of documents successfully inserted. + */ + insertedCount: number; + + /** + * Array of errors that occurred during the write operation. + */ + errors: Array<{ + documentId?: unknown; + error: Error; + }>; +} + +/** + * Interface for writing documents to a target collection. + */ +export interface DocumentWriter { + /** + * Writes documents in bulk to the target collection. + * + * @param connectionId Connection identifier for the target + * @param databaseName Name of the target database + * @param collectionName Name of the target collection + * @param documents Array of documents to write + * @param options Optional write options + * @returns Promise resolving to the write result + */ + writeDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + documents: DocumentDetails[], + options?: DocumentWriterOptions, + ): Promise; + + /** + * Ensures the target collection exists before writing. + * May need methods for pre-flight checks or setup. + * + * @param connectionId Connection identifier for the target + * @param databaseName Name of the target database + * @param collectionName Name of the target collection + * @returns Promise that resolves when the collection is ready + */ + ensureCollectionExists(connectionId: string, databaseName: string, collectionName: string): Promise; +} From fbf169a5acfd5dc044a3645b7ae3b5fae616108d Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Sat, 12 Jul 2025 16:12:54 +0000 Subject: [PATCH 08/21] refine insert result parsing --- .../importDocuments/importDocuments.ts | 17 ++++++-- src/documentdb/ClustersClient.ts | 42 ++++++++++++------- src/documentdb/DocumentProvider.ts | 10 +++-- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/commands/importDocuments/importDocuments.ts b/src/commands/importDocuments/importDocuments.ts index 0013fd1af..df0fac06c 100644 --- a/src/commands/importDocuments/importDocuments.ts +++ b/src/commands/importDocuments/importDocuments.ts @@ -8,7 +8,7 @@ import * as l10n from '@vscode/l10n'; import { EJSON, type Document } from 'bson'; import * as fse from 'fs-extra'; import * as vscode from 'vscode'; -import { ClustersClient } from '../../documentdb/ClustersClient'; +import { ClustersClient, isMongoBulkWriteError } from '../../documentdb/ClustersClient'; import { AzureDomains, getHostsFromConnectionString, @@ -285,10 +285,19 @@ async function insertDocumentWithBufferIntoCluster( // Documents to process could be the current document (if too large) // or the contents of the buffer (if it was full) const client = await ClustersClient.getClient(node.cluster.id); - const insertResult = await client.insertDocuments(databaseName, collectionName, documentsToProcess as Document[]); + const insertResult = await client.insertDocuments( + databaseName, + collectionName, + documentsToProcess as Document[], + false, + ); return { - count: insertResult.insertedCount, - errorOccurred: insertResult.insertedCount < (documentsToProcess?.length || 0), + count: + insertResult.result?.insertedCount ?? + (insertResult.error && isMongoBulkWriteError(insertResult.error) + ? insertResult.error.result?.insertedCount + : 0), + errorOccurred: insertResult.error !== null, }; } diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index 9b3f0ec70..add6f44fd 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -21,6 +21,7 @@ import { type Document, type Filter, type FindOptions, + type InsertManyResult, type ListDatabasesResult, type MongoClientOptions, type WithId, @@ -57,12 +58,9 @@ export interface IndexItemModel { version?: number; } -// Currently we only return insertedCount, but we can add more fields in the future if needed -// Keep the type definition here for future extensibility -export type InsertDocumentsResult = { - /** The number of inserted documents for this operations */ - insertedCount: number; -}; +export function isMongoBulkWriteError(error: unknown): error is MongoBulkWriteError { + return error instanceof MongoBulkWriteError; +} export class ClustersClient { // cache of active/existing clients @@ -475,9 +473,13 @@ export class ClustersClient { databaseName: string, collectionName: string, documents: Document[], - ): Promise { + ordered: boolean = true, + ): Promise<{ result: InsertManyResult | null; error: MongoBulkWriteError | Error | null }> { if (documents.length === 0) { - return { insertedCount: 0 }; + return { + result: { acknowledged: false, insertedIds: {}, insertedCount: 0 }, + error: null, + }; } const collection = this._mongoClient.db(databaseName).collection(collectionName); @@ -487,13 +489,11 @@ export class ClustersClient { // Setting `ordered` to be false allows MongoDB to continue inserting remaining documents even if previous fails. // More details: https://www.mongodb.com/docs/manual/reference/method/db.collection.insertMany/#syntax - ordered: false, + ordered: ordered, }); - return { - insertedCount: insertManyResults.insertedCount, - }; + return { result: insertManyResults, error: null }; } catch (error) { - // print error messages to the console + // Log error messages to the console if (error instanceof MongoBulkWriteError) { const writeErrors: WriteError[] = Array.isArray(error.writeErrors) ? (error.writeErrors as WriteError[]) @@ -510,13 +510,27 @@ export class ClustersClient { ext.outputChannel.appendLog(l10n.t('Write error: {0}', fullErrorMessage)); } ext.outputChannel.show(); + + // Return the error with any partial results + return { + result: null, + error: error, + }; } else if (error instanceof Error) { ext.outputChannel.appendLog(l10n.t('Error: {0}', error.message)); ext.outputChannel.show(); + + // Return the error + return { + result: null, + error: error, + }; } + // Return unknown error return { - insertedCount: error instanceof MongoBulkWriteError ? error.insertedCount || 0 : 0, + result: null, + error: new Error('Unknown error occurred'), }; } } diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts index 0bc56a735..52bce4232 100644 --- a/src/documentdb/DocumentProvider.ts +++ b/src/documentdb/DocumentProvider.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { type Document, type WithId } from 'mongodb'; -import { ClustersClient } from '../documentdb/ClustersClient'; +import { ClustersClient, isMongoBulkWriteError } from '../documentdb/ClustersClient'; import { type BulkWriteResult, type DocumentDetails, @@ -94,10 +94,14 @@ export class MongoDocumentWriter implements DocumentWriter { // Convert DocumentDetails to MongoDB documents const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); - const result = await client.insertDocuments(databaseName, collectionName, mongoDocuments); + const insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments); return { - insertedCount: result.insertedCount, + insertedCount: + insertResult.result?.insertedCount ?? + (insertResult.error && isMongoBulkWriteError(insertResult.error) + ? insertResult.error.result?.insertedCount + : 0), errors: [], // ClustersClient.insertDocuments doesn't return detailed errors in the current implementation }; } catch (error: unknown) { From c1e8552aee52da090284203be496463ccd8ff8a0 Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Sat, 12 Jul 2025 20:49:29 +0000 Subject: [PATCH 09/21] two strategies --- .../importDocuments/importDocuments.ts | 42 ++++++---- .../pasteCollection/pasteCollection.ts | 3 +- src/documentdb/ClustersClient.ts | 48 ++---------- src/documentdb/DocumentProvider.ts | 78 ++++++++++--------- src/services/tasks/CopyPasteCollectionTask.ts | 26 ++++++- src/utils/copyPasteUtils.ts | 14 ++-- 6 files changed, 107 insertions(+), 104 deletions(-) diff --git a/src/commands/importDocuments/importDocuments.ts b/src/commands/importDocuments/importDocuments.ts index df0fac06c..39ef695f5 100644 --- a/src/commands/importDocuments/importDocuments.ts +++ b/src/commands/importDocuments/importDocuments.ts @@ -285,19 +285,31 @@ async function insertDocumentWithBufferIntoCluster( // Documents to process could be the current document (if too large) // or the contents of the buffer (if it was full) const client = await ClustersClient.getClient(node.cluster.id); - const insertResult = await client.insertDocuments( - databaseName, - collectionName, - documentsToProcess as Document[], - false, - ); - - return { - count: - insertResult.result?.insertedCount ?? - (insertResult.error && isMongoBulkWriteError(insertResult.error) - ? insertResult.error.result?.insertedCount - : 0), - errorOccurred: insertResult.error !== null, - }; + try { + const insertResult = await client.insertDocuments( + databaseName, + collectionName, + documentsToProcess as Document[], + false, + ); + return { + count: insertResult.insertedCount, + errorOccurred: false, + }; + } catch (error) { + if (isMongoBulkWriteError(error)) { + // Handle MongoDB bulk write errors + // It could be a partial failure, so we need to check the result + return { + count: error.result.insertedCount, + errorOccurred: true, + }; + } else { + // Handle other errors + return { + count: 0, + errorOccurred: true, + }; + } + } } diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index e81b4d400..fb88303dc 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -74,8 +74,9 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll databaseName: targetNode.databaseInfo.name, collectionName: targetNode.collectionInfo.name, }, - // Currently we only support aborting on conflict + // Currently we only support aborting and skipping on conflict onConflict: ConflictResolutionStrategy.Abort, + // onConflict: ConflictResolutionStrategy.Skip, }; // Create task with documentDB document providers diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index add6f44fd..dbd07f837 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -26,10 +26,8 @@ import { type MongoClientOptions, type WithId, type WithoutId, - type WriteError, } from 'mongodb'; import { Links } from '../constants'; -import { ext } from '../extensionVariables'; import { type EmulatorConfiguration } from '../utils/emulatorConfiguration'; import { CredentialCache } from './CredentialCache'; import { getHostsFromConnectionString, hasAzureDomain } from './utils/connectionStringHelpers'; @@ -474,12 +472,9 @@ export class ClustersClient { collectionName: string, documents: Document[], ordered: boolean = true, - ): Promise<{ result: InsertManyResult | null; error: MongoBulkWriteError | Error | null }> { + ): Promise { if (documents.length === 0) { - return { - result: { acknowledged: false, insertedIds: {}, insertedCount: 0 }, - error: null, - }; + return { acknowledged: false, insertedIds: {}, insertedCount: 0 }; } const collection = this._mongoClient.db(databaseName).collection(collectionName); @@ -491,47 +486,16 @@ export class ClustersClient { // More details: https://www.mongodb.com/docs/manual/reference/method/db.collection.insertMany/#syntax ordered: ordered, }); - return { result: insertManyResults, error: null }; + return insertManyResults; } catch (error) { // Log error messages to the console if (error instanceof MongoBulkWriteError) { - const writeErrors: WriteError[] = Array.isArray(error.writeErrors) - ? (error.writeErrors as WriteError[]) - : [error.writeErrors as WriteError]; - - for (const writeError of writeErrors) { - const generalErrorMessage = parseError(writeError).message; - const descriptiveErrorMessage = writeError.err?.errmsg; - - const fullErrorMessage = descriptiveErrorMessage - ? `${generalErrorMessage} - ${descriptiveErrorMessage}` - : generalErrorMessage; - - ext.outputChannel.appendLog(l10n.t('Write error: {0}', fullErrorMessage)); - } - ext.outputChannel.show(); - - // Return the error with any partial results - return { - result: null, - error: error, - }; + throw error; } else if (error instanceof Error) { - ext.outputChannel.appendLog(l10n.t('Error: {0}', error.message)); - ext.outputChannel.show(); - - // Return the error - return { - result: null, - error: error, - }; + throw error; } - // Return unknown error - return { - result: null, - error: new Error('Unknown error occurred'), - }; + throw new Error(l10n.t('An unknown error occurred while inserting documents.')); } } } diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts index 52bce4232..94df9a970 100644 --- a/src/documentdb/DocumentProvider.ts +++ b/src/documentdb/DocumentProvider.ts @@ -3,10 +3,12 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { type Document, type WithId } from 'mongodb'; +import { type Document, type InsertManyResult, type WithId, type WriteError } from 'mongodb'; import { ClustersClient, isMongoBulkWriteError } from '../documentdb/ClustersClient'; import { + ConflictResolutionStrategy, type BulkWriteResult, + type CopyPasteConfig, type DocumentDetails, type DocumentReader, type DocumentWriter, @@ -79,6 +81,7 @@ export class MongoDocumentWriter implements DocumentWriter { connectionId: string, databaseName: string, collectionName: string, + config: CopyPasteConfig, documents: DocumentDetails[], _options?: DocumentWriterOptions, ): Promise { @@ -94,49 +97,48 @@ export class MongoDocumentWriter implements DocumentWriter { // Convert DocumentDetails to MongoDB documents const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); - const insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments); + let insertResult: InsertManyResult; + + switch (config.onConflict) { + case ConflictResolutionStrategy.Skip: + insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments, false); + break; + case ConflictResolutionStrategy.Abort: + insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments, false); + break; + default: + throw new Error(`Unsupported conflict resolution strategy: ${config.onConflict}`); + } return { - insertedCount: - insertResult.result?.insertedCount ?? - (insertResult.error && isMongoBulkWriteError(insertResult.error) - ? insertResult.error.result?.insertedCount - : 0), - errors: [], // ClustersClient.insertDocuments doesn't return detailed errors in the current implementation + insertedCount: insertResult.insertedCount, + errors: null, // MongoDB bulk write errors will be handled in the catch block }; } catch (error: unknown) { - // Handle MongoDB bulk write errors - const errors: Array<{ documentId?: unknown; error: Error }> = []; - - if (error && typeof error === 'object' && 'writeErrors' in error) { - const writeErrors = (error as { writeErrors: unknown[] }).writeErrors; - for (const writeError of writeErrors) { - if (writeError && typeof writeError === 'object' && 'index' in writeError) { - const docIndex = writeError.index as number; - const documentId = docIndex < documents.length ? documents[docIndex].id : undefined; - const errorMessage = - 'errmsg' in writeError ? (writeError.errmsg as string) : 'Unknown write error'; - errors.push({ - documentId, - error: new Error(errorMessage), - }); - } - } + if (isMongoBulkWriteError(error)) { + // Handle MongoDB bulk write errors + const writeErrorsArray = ( + Array.isArray(error.writeErrors) ? error.writeErrors : [error.writeErrors] + ) as Array; + return { + insertedCount: error.result.insertedCount, + errors: writeErrorsArray.map((writeError) => ({ + documentId: (writeError.getOperation()._id as string) || undefined, + error: new Error(writeError.errmsg || 'Unknown write error'), + })), + }; + } else if (error instanceof Error) { + return { + insertedCount: 0, + errors: [{ documentId: undefined, error }], + }; } else { - errors.push({ - error: error instanceof Error ? error : new Error(String(error)), - }); + // Handle unknown error types + return { + insertedCount: 0, + errors: [{ documentId: undefined, error: new Error(String(error)) }], + }; } - - const insertedCount = - error && typeof error === 'object' && 'result' in error - ? ((error as { result?: { insertedCount?: number } }).result?.insertedCount ?? 0) - : 0; - - return { - insertedCount, - errors, - }; } } diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts index a422cc440..e9a1d1def 100644 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as vscode from 'vscode'; +import { ext } from '../../extensionVariables'; import { type CopyPasteConfig, type DocumentDetails, @@ -185,6 +186,7 @@ export class CopyPasteCollectionTask extends Task { this.config.target.connectionId, this.config.target.databaseName, this.config.target.collectionName, + this.config, buffer, { batchSize: buffer.length }, ); @@ -193,11 +195,29 @@ export class CopyPasteCollectionTask extends Task { this.processedDocuments += result.insertedCount; // Check for errors in the write result - if (result.errors.length > 0) { + if (result.errors !== null) { // For basic implementation with abort strategy, any error should fail the task if (this.config.onConflict === ConflictResolutionStrategy.Abort) { - const firstError = result.errors[0]; - throw new Error(vscode.l10n.t('Write operation failed: {0}', firstError.error.message)); + const firstError = result.errors[0] as { error: Error }; + throw new Error( + vscode.l10n.t( + 'Task aborted because of error: {0}, {1} document(s) were inserted in total', + firstError.error?.message ?? 'Unknown error', + this.processedDocuments.toString(), + ), + ); + } else if (this.config.onConflict === ConflictResolutionStrategy.Skip) { + // For skip strategy, we can log the errors but continue + for (const error of result.errors) { + ext.outputChannel.appendLog( + vscode.l10n.t( + 'Skipped document {0} due to error: {1}', + String(error.documentId ?? 'unknown'), + error.error?.message ?? 'Unknown error', + ), + ); + } + ext.outputChannel.show(); } // Future: Handle other conflict resolution strategies } diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts index 48e6dc9cc..16e3ec150 100644 --- a/src/utils/copyPasteUtils.ts +++ b/src/utils/copyPasteUtils.ts @@ -11,7 +11,12 @@ export enum ConflictResolutionStrategy { * Abort the operation if any conflict or error occurs */ Abort = 'abort', - // Future options: Overwrite = 'overwrite', Skip = 'skip' + + /** + * Skip the conflicting document and continue with the operation + */ + Skip = 'skip', + // Future options: Overwrite = 'overwrite' } /** @@ -114,10 +119,8 @@ export interface BulkWriteResult { /** * Array of errors that occurred during the write operation. */ - errors: Array<{ - documentId?: unknown; - error: Error; - }>; + errors: Array<{ documentId?: string; error: Error }> | null; // Should be typed more specifically based on the implementation + // e.g., for MongoDB, this could be an array of MongoBulkWriteError objects } /** @@ -138,6 +141,7 @@ export interface DocumentWriter { connectionId: string, databaseName: string, collectionName: string, + config: CopyPasteConfig, documents: DocumentDetails[], options?: DocumentWriterOptions, ): Promise; From b721aaf4cbd4e4a63f9842ef0cb906608b0b0d20 Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Sat, 12 Jul 2025 20:59:32 +0000 Subject: [PATCH 10/21] l10n --- l10n/bundle.l10n.json | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index 53cd139dd..93ed77eaa 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -47,6 +47,7 @@ "An element with the following id already exists: {id}": "An element with the following id already exists: {id}", "An error has occurred. Check output window for more details.": "An error has occurred. Check output window for more details.", "An item with id \"{0}\" already exists for workspace \"{1}\".": "An item with id \"{0}\" already exists for workspace \"{1}\".", + "An unknown error occurred while inserting documents.": "An unknown error occurred while inserting documents.", "API version \"{0}\" for extension id \"{1}\" is no longer supported. Minimum version is \"{2}\".": "API version \"{0}\" for extension id \"{1}\" is no longer supported. Minimum version is \"{2}\".", "API: Registered new migration provider: \"{providerId}\" - \"{providerLabel}\"": "API: Registered new migration provider: \"{providerId}\" - \"{providerLabel}\"", "Are you sure?": "Are you sure?", @@ -82,6 +83,7 @@ "Click here to update credentials": "Click here to update credentials", "Click to view resource": "Click to view resource", "Collection \"{0}\" from database \"{1}\" has been marked for copy.": "Collection \"{0}\" from database \"{1}\" has been marked for copy.", + "Collection copied successfully: {0}": "Collection copied successfully: {0}", "Collection name cannot begin with the system. prefix (Reserved for internal use).": "Collection name cannot begin with the system. prefix (Reserved for internal use).", "Collection name cannot contain .system.": "Collection name cannot contain .system.", "Collection name cannot contain the $.": "Collection name cannot contain the $.", @@ -99,10 +101,18 @@ "Connection string is not set": "Connection string is not set", "Connection string not found.": "Connection string not found.", "Connection updated successfully.": "Connection updated successfully.", + "Copied {0} of {1} documents": "Copied {0} of {1} documents", + "Copy": "Copy", + "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.": "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.", + "Copy collection \"{0}\" from \"{1}\" to \"{2}\"": "Copy collection \"{0}\" from \"{1}\" to \"{2}\"", + "Copy operation completed successfully": "Copy operation completed successfully", + "Copy operation failed: {0}": "Copy operation failed: {0}", + "Copy operation was cancelled.": "Copy operation was cancelled.", "CosmosDB Accounts": "CosmosDB Accounts", "Could not find {0}": "Could not find {0}", "Could not find the Azure Resource Groups extension": "Could not find the Azure Resource Groups extension", "Could not find unique name for new file.": "Could not find unique name for new file.", + "Counting documents in source collection...": "Counting documents in source collection...", "Create an Azure Account...": "Create an Azure Account...", "Create an Azure for Students Account...": "Create an Azure for Students Account...", "Create collection": "Create collection", @@ -150,6 +160,7 @@ "Element with id of {rootId} not found.": "Element with id of {rootId} not found.", "Enable TLS/SSL (Default)": "Enable TLS/SSL (Default)", "Enforce TLS/SSL checks for a secure connection.": "Enforce TLS/SSL checks for a secure connection.", + "Ensuring target collection exists...": "Ensuring target collection exists...", "Enter a collection name.": "Enter a collection name.", "Enter a database name.": "Enter a database name.", "Enter the Azure VM tag key used for discovering DocumentDB instances.": "Enter the Azure VM tag key used for discovering DocumentDB instances.", @@ -194,12 +205,15 @@ "Extension dependency with id \"{0}\" must be updated.": "Extension dependency with id \"{0}\" must be updated.", "Failed to connect to \"{cluster}\"": "Failed to connect to \"{cluster}\"", "Failed to connect to VM \"{vmName}\"": "Failed to connect to VM \"{vmName}\"", + "Failed to copy collection: {0}": "Failed to copy collection: {0}", + "Failed to count documents in source collection: {0}": "Failed to count documents in source collection: {0}", "Failed to create Azure management clients: {0}": "Failed to create Azure management clients: {0}", "Failed to create role assignment \"{0}\" for the {2} resource \"{1}\".": "Failed to create role assignment \"{0}\" for the {2} resource \"{1}\".", "Failed to create role assignment(s).": "Failed to create role assignment(s).", "Failed to delete documents. Unknown error.": "Failed to delete documents. Unknown error.", "Failed to delete item \"{0}\".": "Failed to delete item \"{0}\".", "Failed to delete secrets for item \"{0}\".": "Failed to delete secrets for item \"{0}\".", + "Failed to ensure target collection exists: {0}": "Failed to ensure target collection exists: {0}", "Failed to export documents. Please see the output for details.": "Failed to export documents. Please see the output for details.", "Failed to extract the connection string from the selected account.": "Failed to extract the connection string from the selected account.", "Failed to extract the connection string from the selected node.": "Failed to extract the connection string from the selected node.", @@ -257,6 +271,7 @@ "Invalid connection type selected.": "Invalid connection type selected.", "Invalid document ID: {0}": "Invalid document ID: {0}", "Invalid semver \"{0}\".": "Invalid semver \"{0}\".", + "Invalid source or target node type.": "Invalid source or target node type.", "Invalid workspace resource ID: {0}": "Invalid workspace resource ID: {0}", "JSON View": "JSON View", "Learn more": "Learn more", @@ -301,6 +316,7 @@ "No commands found in this document.": "No commands found in this document.", "No Connectivity": "No Connectivity", "No credentials found for id {credentialId}": "No credentials found for id {credentialId}", + "No documents to copy. Operation completed.": "No documents to copy. Operation completed.", "No matching resources found.": "No matching resources found.", "No node selected.": "No node selected.", "No properties found in the schema at path \"{0}\"": "No properties found in the schema at path \"{0}\"", @@ -309,6 +325,7 @@ "No scope was provided for the role assignment.": "No scope was provided for the role assignment.", "No session found for id {sessionId}": "No session found for id {sessionId}", "No subscriptions found": "No subscriptions found", + "No target node selected.": "No target node selected.", "Not connected to any MongoDB database.": "Not connected to any MongoDB database.", "Note: This confirmation type can be configured in the extension settings.": "Note: This confirmation type can be configured in the extension settings.", "Note: You can disable these URL handling confirmations in the extension settings.": "Note: You can disable these URL handling confirmations in the extension settings.", @@ -374,11 +391,13 @@ "Signing out programmatically is not supported. You must sign out by selecting the account in the Accounts menu and choosing Sign Out.": "Signing out programmatically is not supported. You must sign out by selecting the account in the Accounts menu and choosing Sign Out.", "Simulated failure at step {0} for testing purposes": "Simulated failure at step {0} for testing purposes", "Skip for now": "Skip for now", + "Skipped document {0} due to error: {1}": "Skipped document {0} due to error: {1}", "Small breadcrumb example with buttons": "Small breadcrumb example with buttons", "Some items could not be displayed": "Some items could not be displayed", "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "Specified character lengths should be 1 character or greater.": "Specified character lengths should be 1 character or greater.", "Started executable: \"{command}\". Connecting to host…": "Started executable: \"{command}\". Connecting to host…", + "Starting document copy...": "Starting document copy...", "Starting executable: \"{command}\"": "Starting executable: \"{command}\"", "Starts with mongodb:// or mongodb+srv://": "Starts with mongodb:// or mongodb+srv://", "Stopping {0}": "Stopping {0}", @@ -392,6 +411,8 @@ "Tag can only contain alphanumeric characters, underscores, periods, and hyphens.": "Tag can only contain alphanumeric characters, underscores, periods, and hyphens.", "Tag cannot be empty.": "Tag cannot be empty.", "Tag cannot be longer than 256 characters.": "Tag cannot be longer than 256 characters.", + "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", + "Task aborted because of error: {0}, {1} document(s) were inserted in total": "Task aborted because of error: {0}, {1} document(s) were inserted in total", "Task completed successfully": "Task completed successfully", "Task created and ready to start": "Task created and ready to start", "Task failed": "Task failed", @@ -402,7 +423,6 @@ "Task will fail at a random step for testing": "Task will fail at a random step for testing", "Task with ID {0} already exists": "Task with ID {0} already exists", "Task with ID {0} not found": "Task with ID {0} not found", - "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "The \"{databaseId}\" database has been deleted.": "The \"{databaseId}\" database has been deleted.", "The \"{name}\" database has been created.": "The \"{name}\" database has been created.", "The \"{newCollectionName}\" collection has been created.": "The \"{newCollectionName}\" collection has been created.", @@ -483,7 +503,6 @@ "with Popover": "with Popover", "Working…": "Working…", "Would you like to open the Collection View?": "Would you like to open the Collection View?", - "Write error: {0}": "Write error: {0}", "Yes": "Yes", "Yes, continue": "Yes, continue", "Yes, open Collection View": "Yes, open Collection View", From 2458adc6dcdd8a991f2100936ad5efd99bc3507d Mon Sep 17 00:00:00 2001 From: xingfan-git Date: Tue, 15 Jul 2025 07:30:53 +0000 Subject: [PATCH 11/21] overwrite --- l10n/bundle.l10n.json | 11 ++- .../pasteCollection/pasteCollection.ts | 3 +- src/documentdb/ClustersClient.ts | 72 +++++++++++++++++++ src/documentdb/DocumentProvider.ts | 57 ++++++++++----- src/services/tasks/CopyPasteCollectionTask.ts | 8 ++- src/utils/copyPasteUtils.ts | 6 +- 6 files changed, 135 insertions(+), 22 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index 93ed77eaa..47e2c3ce0 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -203,6 +203,8 @@ "Exporting documents": "Exporting documents", "Exporting…": "Exporting…", "Extension dependency with id \"{0}\" must be updated.": "Extension dependency with id \"{0}\" must be updated.", + "Failed to abort transaction: {0}": "Failed to abort transaction: {0}", + "Failed to commit transaction: {0}": "Failed to commit transaction: {0}", "Failed to connect to \"{cluster}\"": "Failed to connect to \"{cluster}\"", "Failed to connect to VM \"{vmName}\"": "Failed to connect to VM \"{vmName}\"", "Failed to copy collection: {0}": "Failed to copy collection: {0}", @@ -213,19 +215,25 @@ "Failed to delete documents. Unknown error.": "Failed to delete documents. Unknown error.", "Failed to delete item \"{0}\".": "Failed to delete item \"{0}\".", "Failed to delete secrets for item \"{0}\".": "Failed to delete secrets for item \"{0}\".", + "Failed to end session: {0}": "Failed to end session: {0}", "Failed to ensure target collection exists: {0}": "Failed to ensure target collection exists: {0}", "Failed to export documents. Please see the output for details.": "Failed to export documents. Please see the output for details.", "Failed to extract the connection string from the selected account.": "Failed to extract the connection string from the selected account.", "Failed to extract the connection string from the selected node.": "Failed to extract the connection string from the selected node.", "Failed to find commandId on generic tree item.": "Failed to find commandId on generic tree item.", + "Failed to get collection {0} in database {1}: {2}": "Failed to get collection {0} in database {1}: {2}", "Failed to get public IP": "Failed to get public IP", "Failed to initialize Azure management clients": "Failed to initialize Azure management clients", "Failed to initialize task": "Failed to initialize task", + "Failed to overwrite documents: {0}": "Failed to overwrite documents: {0}", "Failed to parse secrets for key {0}:": "Failed to parse secrets for key {0}:", "Failed to process URI: {0}": "Failed to process URI: {0}", "Failed to rename the connection.": "Failed to rename the connection.", "Failed to save credentials for \"{cluster}\".": "Failed to save credentials for \"{cluster}\".", "Failed to save credentials.": "Failed to save credentials.", + "Failed to start a session: {0}": "Failed to start a session: {0}", + "Failed to start a transaction with the provided session: {0}": "Failed to start a transaction with the provided session: {0}", + "Failed to start a transaction: {0}": "Failed to start a transaction: {0}", "Failed to store secrets for key {0}:": "Failed to store secrets for key {0}:", "Failed to update the connection.": "Failed to update the connection.", "Failed with code \"{0}\".": "Failed with code \"{0}\".", @@ -391,7 +399,7 @@ "Signing out programmatically is not supported. You must sign out by selecting the account in the Accounts menu and choosing Sign Out.": "Signing out programmatically is not supported. You must sign out by selecting the account in the Accounts menu and choosing Sign Out.", "Simulated failure at step {0} for testing purposes": "Simulated failure at step {0} for testing purposes", "Skip for now": "Skip for now", - "Skipped document {0} due to error: {1}": "Skipped document {0} due to error: {1}", + "Skipped document with _id: {0} due to error: {1}": "Skipped document with _id: {0} due to error: {1}", "Small breadcrumb example with buttons": "Small breadcrumb example with buttons", "Some items could not be displayed": "Some items could not be displayed", "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}", @@ -416,6 +424,7 @@ "Task completed successfully": "Task completed successfully", "Task created and ready to start": "Task created and ready to start", "Task failed": "Task failed", + "Task failed due to error: {0}": "Task failed due to error: {0}", "Task is running": "Task is running", "Task stopped": "Task stopped", "Task stopped during initialization": "Task stopped during initialization", diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index fb88303dc..235f1d5ee 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -75,8 +75,9 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll collectionName: targetNode.collectionInfo.name, }, // Currently we only support aborting and skipping on conflict - onConflict: ConflictResolutionStrategy.Abort, + // onConflict: ConflictResolutionStrategy.Abort, // onConflict: ConflictResolutionStrategy.Skip, + onConflict: ConflictResolutionStrategy.Overwrite, }; // Create task with documentDB document providers diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index dbd07f837..ef4a9e12d 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -16,6 +16,7 @@ import { MongoBulkWriteError, MongoClient, ObjectId, + type ClientSession, type Collection, type DeleteResult, type Document, @@ -193,6 +194,62 @@ export class ClustersClient { } } + startTransaction(): ClientSession { + try { + const session = this._mongoClient.startSession(); + session.startTransaction(); + return session; + } catch (error) { + throw new Error(l10n.t('Failed to start a transaction: {0}', parseError(error).message)); + } + } + + startTransactionWithSession(session: ClientSession): void { + try { + session.startTransaction(); + } catch (error) { + throw new Error( + l10n.t('Failed to start a transaction with the provided session: {0}', parseError(error).message), + ); + } + } + + async commitTransaction(session: ClientSession): Promise { + try { + await session.commitTransaction(); + } catch (error) { + throw new Error(l10n.t('Failed to commit transaction: {0}', parseError(error).message)); + } finally { + this.endSession(session); + } + } + + async abortTransaction(session: ClientSession): Promise { + try { + await session.abortTransaction(); + } catch (error) { + throw new Error(l10n.t('Failed to abort transaction: {0}', parseError(error).message)); + } finally { + this.endSession(session); + } + } + + startSession(): ClientSession { + try { + return this._mongoClient.startSession(); + } catch (error) { + throw new Error(l10n.t('Failed to start a session: {0}', parseError(error).message)); + } + } + + endSession(session: ClientSession): void { + try { + void session.endSession(); + } catch (error) { + throw new Error(l10n.t('Failed to end session: {0}', parseError(error).message)); + } + } + getUserName() { return CredentialCache.getCredentials(this.credentialId)?.connectionUser; } @@ -204,6 +261,21 @@ export class ClustersClient { return CredentialCache.getConnectionStringWithPassword(this.credentialId); } + getCollection(databaseName: string, collectionName: string): Collection { + try { + return this._mongoClient.db(databaseName).collection(collectionName); + } catch (error) { + throw new Error( + l10n.t( + 'Failed to get collection {0} in database {1}: {2}', + collectionName, + databaseName, + parseError(error).message, + ), + ); + } + } + async listDatabases(): Promise { const rawDatabases: ListDatabasesResult = await this._mongoClient.db().admin().listDatabases(); const databases: DatabaseItemModel[] = rawDatabases.databases.filter( diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts index 94df9a970..d3d2a4521 100644 --- a/src/documentdb/DocumentProvider.ts +++ b/src/documentdb/DocumentProvider.ts @@ -3,7 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { type Document, type InsertManyResult, type WithId, type WriteError } from 'mongodb'; +import { parseError } from '@microsoft/vscode-azext-utils'; +import { type Document, type ObjectId, type WithId, type WriteError } from 'mongodb'; +import { l10n } from 'vscode'; import { ClustersClient, isMongoBulkWriteError } from '../documentdb/ClustersClient'; import { ConflictResolutionStrategy, @@ -92,23 +94,21 @@ export class MongoDocumentWriter implements DocumentWriter { }; } - try { - const client = await ClustersClient.getClient(connectionId); + const client = await ClustersClient.getClient(connectionId); - // Convert DocumentDetails to MongoDB documents - const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); - let insertResult: InsertManyResult; + // Convert DocumentDetails to MongoDB documents + const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); - switch (config.onConflict) { - case ConflictResolutionStrategy.Skip: - insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments, false); - break; - case ConflictResolutionStrategy.Abort: - insertResult = await client.insertDocuments(databaseName, collectionName, mongoDocuments, false); - break; - default: - throw new Error(`Unsupported conflict resolution strategy: ${config.onConflict}`); - } + try { + const insertResult = await client.insertDocuments( + databaseName, + collectionName, + mongoDocuments, + // For abort on conflict, we set ordered to true to make it throw on the first error + // For skip on conflict, we set ordered to false + // For overwrite on conflict, we use ordered as a filter to find documents that should be overwritten + config.onConflict === ConflictResolutionStrategy.Abort, + ); return { insertedCount: insertResult.insertedCount, @@ -116,10 +116,33 @@ export class MongoDocumentWriter implements DocumentWriter { }; } catch (error: unknown) { if (isMongoBulkWriteError(error)) { - // Handle MongoDB bulk write errors const writeErrorsArray = ( Array.isArray(error.writeErrors) ? error.writeErrors : [error.writeErrors] ) as Array; + + if (config.onConflict === ConflictResolutionStrategy.Overwrite) { + // For overwrite strategy, we need to delete the conflicting documents and then re-insert + const session = client.startTransaction(); + const collection = client.getCollection(databaseName, collectionName); + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const idsToOverwrite = writeErrorsArray.map((we) => we.getOperation()._id) as Array; + const documentsToOverwrite = mongoDocuments.filter((doc) => + idsToOverwrite.includes((doc as WithId)._id as ObjectId), + ); + await collection.deleteMany({ _id: { $in: idsToOverwrite } }, { session }); + const insertResult = await collection.insertMany(documentsToOverwrite, { session }); + await client.commitTransaction(session); + return { + insertedCount: insertResult.insertedCount, + errors: null, + }; + } catch (error) { + await client.abortTransaction(session); + throw new Error(l10n.t('Failed to overwrite documents: {0}', parseError(error).message)); + } + } + return { insertedCount: error.result.insertedCount, errors: writeErrorsArray.map((writeError) => ({ diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts index e9a1d1def..5e4358c49 100644 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -211,15 +211,19 @@ export class CopyPasteCollectionTask extends Task { for (const error of result.errors) { ext.outputChannel.appendLog( vscode.l10n.t( - 'Skipped document {0} due to error: {1}', + 'Skipped document with _id: {0} due to error: {1}', String(error.documentId ?? 'unknown'), error.error?.message ?? 'Unknown error', ), ); } ext.outputChannel.show(); + } else { + ext.outputChannel.appendLog( + vscode.l10n.t('Task failed due to error: {0}', result.errors[0].error?.message ?? 'Unknown error'), + ); + ext.outputChannel.show(); } - // Future: Handle other conflict resolution strategies } // Update progress diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts index 16e3ec150..13132c930 100644 --- a/src/utils/copyPasteUtils.ts +++ b/src/utils/copyPasteUtils.ts @@ -16,7 +16,11 @@ export enum ConflictResolutionStrategy { * Skip the conflicting document and continue with the operation */ Skip = 'skip', - // Future options: Overwrite = 'overwrite' + + /** + * Overwrite the existing document in the target collection with the source document + */ + Overwrite = 'overwrite', } /** From d8eff8d690a761b9fbf4f8791dbea904cd850869 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 13:53:38 +0200 Subject: [PATCH 12/21] Updated copy-paste-task, minor changes + update to final state reporting --- .../importDocuments/importDocuments.ts | 4 +- .../pasteCollection/pasteCollection.ts | 76 ++++++++++--------- src/documentdb/ClustersClient.ts | 3 +- src/documentdb/DocumentProvider.ts | 42 +++++----- src/services/tasks/CopyPasteCollectionTask.ts | 27 +++---- src/utils/copyPasteUtils.ts | 9 +-- 6 files changed, 84 insertions(+), 77 deletions(-) diff --git a/src/commands/importDocuments/importDocuments.ts b/src/commands/importDocuments/importDocuments.ts index d3a8961d9..7742e98be 100644 --- a/src/commands/importDocuments/importDocuments.ts +++ b/src/commands/importDocuments/importDocuments.ts @@ -8,7 +8,7 @@ import * as l10n from '@vscode/l10n'; import { EJSON, type Document } from 'bson'; import * as fs from 'node:fs/promises'; import * as vscode from 'vscode'; -import { ClustersClient, isMongoBulkWriteError } from '../../documentdb/ClustersClient'; +import { ClustersClient, isBulkWriteError } from '../../documentdb/ClustersClient'; import { AzureDomains, getHostsFromConnectionString, @@ -297,7 +297,7 @@ async function insertDocumentWithBufferIntoCluster( errorOccurred: false, }; } catch (error) { - if (isMongoBulkWriteError(error)) { + if (isBulkWriteError(error)) { // Handle MongoDB bulk write errors // It could be a partial failure, so we need to check the result return { diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 235f1d5ee..8cf1a65a1 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -6,7 +6,7 @@ import { type IActionContext } from '@microsoft/vscode-azext-utils'; import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; -import { MongoDocumentReader, MongoDocumentWriter } from '../../documentdb/DocumentProvider'; +import { DocumentDbDocumentReader, DocumentDbDocumentWriter } from '../../documentdb/DocumentProvider'; import { ext } from '../../extensionVariables'; import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; import { TaskService, TaskState } from '../../services/taskService'; @@ -82,9 +82,9 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll // Create task with documentDB document providers // Need to check reader and writer implementations before creating the task - // For now, we only support MongoDB collections - const reader = new MongoDocumentReader(); - const writer = new MongoDocumentWriter(); + // For now, we only support DocumentDB collections + const reader = new DocumentDbDocumentReader(); + const writer = new DocumentDbDocumentWriter(); const task = new CopyPasteCollectionTask(config, reader, writer); // // Get total number of documents in the source collection @@ -97,42 +97,46 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll // Register task with the task service TaskService.registerTask(task); - // Start and monitor the task without showing a progress notification - await task.start(); + task.onDidChangeState((event) => { + if (event.newState === TaskState.Completed) { + const summary = task.getStatus(); + ext.outputChannel.appendLine( + l10n.t("✅ Task '{taskName}' completed successfully. {message}", { + taskName: task.name, + message: summary.message || '', + }), + ); + } else if (event.newState === TaskState.Stopped) { + ext.outputChannel.appendLine( + l10n.t("⏹️ Task '{taskName}' was stopped. {message}", { + taskName: task.name, + message: task.getStatus().message || '', + }), + ); + } else if (event.newState === TaskState.Failed) { + const summary = task.getStatus(); + ext.outputChannel.appendLine( + l10n.t("⚠️ Task '{taskName}' failed. {message}", { + taskName: task.name, + message: summary.message || '', + }), + ); + } + }); - // Wait for the task to complete - while (task.getStatus().state === TaskState.Running || task.getStatus().state === TaskState.Initializing) { - // Simple polling with a small delay - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - // Check final status and show result - const finalStatus = task.getStatus(); - if (finalStatus.state === TaskState.Completed) { - void vscode.window.showInformationMessage( - l10n.t('Collection copied successfully: {0}', finalStatus.message || ''), - ); - } else if (finalStatus.state === TaskState.Failed) { - const errorToThrow = - finalStatus.error instanceof Error ? finalStatus.error : new Error('Copy operation failed'); - throw errorToThrow; - } else if (finalStatus.state === TaskState.Stopped) { - void vscode.window.showInformationMessage(l10n.t('Copy operation was cancelled.')); - } + ext.outputChannel.appendLine(l10n.t("▶️ Task '{taskName}' starting...", { taskName: 'Copy Collection' })); + + // Start the copy-paste task + await task.start(); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); + ext.outputChannel.appendLine( + l10n.t('⚠️ Task failed. {errorMessage}', { + errorMessage: errorMessage, + }), + ); + throw error; - } finally { - // Clean up - remove the task from the service after completion - try { - const task = TaskService.listTasks().find((t) => t.type === 'copy-paste-collection'); - if (task) { - await TaskService.deleteTask(task.id); - } - } catch (cleanupError) { - // Log cleanup error but don't throw - console.warn('Failed to clean up copy-paste task:', cleanupError); - } } } diff --git a/src/documentdb/ClustersClient.ts b/src/documentdb/ClustersClient.ts index 97fd3dd62..fa15e3998 100644 --- a/src/documentdb/ClustersClient.ts +++ b/src/documentdb/ClustersClient.ts @@ -57,7 +57,7 @@ export interface IndexItemModel { version?: number; } -export function isMongoBulkWriteError(error: unknown): error is MongoBulkWriteError { +export function isBulkWriteError(error: unknown): error is MongoBulkWriteError { return error instanceof MongoBulkWriteError; } @@ -354,7 +354,6 @@ export class ClustersClient { } async countDocuments(databaseName: string, collectionName: string, findQuery: string = '{}'): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment if (findQuery === undefined || findQuery.trim().length === 0) { findQuery = '{}'; } diff --git a/src/documentdb/DocumentProvider.ts b/src/documentdb/DocumentProvider.ts index d3d2a4521..b08444133 100644 --- a/src/documentdb/DocumentProvider.ts +++ b/src/documentdb/DocumentProvider.ts @@ -6,7 +6,7 @@ import { parseError } from '@microsoft/vscode-azext-utils'; import { type Document, type ObjectId, type WithId, type WriteError } from 'mongodb'; import { l10n } from 'vscode'; -import { ClustersClient, isMongoBulkWriteError } from '../documentdb/ClustersClient'; +import { ClustersClient, isBulkWriteError } from '../documentdb/ClustersClient'; import { ConflictResolutionStrategy, type BulkWriteResult, @@ -18,13 +18,13 @@ import { } from '../utils/copyPasteUtils'; /** - * MongoDB-specific implementation of DocumentReader. + * DocumentDB-specific implementation of DocumentReader. */ -export class MongoDocumentReader implements DocumentReader { +export class DocumentDbDocumentReader implements DocumentReader { /** - * Streams documents from a MongoDB collection. + * Streams documents from a DocumentDB collection. * - * @param connectionId Connection identifier to get the MongoDB client + * @param connectionId Connection identifier to get the DocumentDB client * @param databaseName Name of the database * @param collectionName Name of the collection * @returns AsyncIterable of document details @@ -46,9 +46,9 @@ export class MongoDocumentReader implements DocumentReader { } /** - * Counts the total number of documents in a MongoDB collection. + * Counts the total number of documents in the DocumentDB collection. * - * @param connectionId Connection identifier to get the MongoDB client + * @param connectionId Connection identifier to get the DocumentDB client * @param databaseName Name of the database * @param collectionName Name of the collection, * @param filter Optional filter to apply to the count operation (default is '{}') @@ -66,13 +66,13 @@ export class MongoDocumentReader implements DocumentReader { } /** - * MongoDB-specific implementation of DocumentWriter. + * DocumentDB-specific implementation of DocumentWriter. */ -export class MongoDocumentWriter implements DocumentWriter { +export class DocumentDbDocumentWriter implements DocumentWriter { /** - * Writes documents to a MongoDB collection using bulk operations. + * Writes documents to a DocumentDB collection using bulk operations. * - * @param connectionId Connection identifier to get the MongoDB client + * @param connectionId Connection identifier to get the DocumentDB client * @param databaseName Name of the target database * @param collectionName Name of the target collection * @param documents Array of documents to write @@ -96,14 +96,14 @@ export class MongoDocumentWriter implements DocumentWriter { const client = await ClustersClient.getClient(connectionId); - // Convert DocumentDetails to MongoDB documents - const mongoDocuments = documents.map((doc) => doc.documentContent as WithId); + // Convert DocumentDetails to DocumentDB documents + const rawDocuments = documents.map((doc) => doc.documentContent as WithId); try { const insertResult = await client.insertDocuments( databaseName, collectionName, - mongoDocuments, + rawDocuments, // For abort on conflict, we set ordered to true to make it throw on the first error // For skip on conflict, we set ordered to false // For overwrite on conflict, we use ordered as a filter to find documents that should be overwritten @@ -112,10 +112,10 @@ export class MongoDocumentWriter implements DocumentWriter { return { insertedCount: insertResult.insertedCount, - errors: null, // MongoDB bulk write errors will be handled in the catch block + errors: null, // DocumentDB bulk write errors will be handled in the catch block }; } catch (error: unknown) { - if (isMongoBulkWriteError(error)) { + if (isBulkWriteError(error)) { const writeErrorsArray = ( Array.isArray(error.writeErrors) ? error.writeErrors : [error.writeErrors] ) as Array; @@ -127,7 +127,7 @@ export class MongoDocumentWriter implements DocumentWriter { try { // eslint-disable-next-line @typescript-eslint/no-unsafe-return const idsToOverwrite = writeErrorsArray.map((we) => we.getOperation()._id) as Array; - const documentsToOverwrite = mongoDocuments.filter((doc) => + const documentsToOverwrite = rawDocuments.filter((doc) => idsToOverwrite.includes((doc as WithId)._id as ObjectId), ); await collection.deleteMany({ _id: { $in: idsToOverwrite } }, { session }); @@ -166,9 +166,9 @@ export class MongoDocumentWriter implements DocumentWriter { } /** - * Ensures the target collection exists in MongoDB. + * Ensures the target collection exists. * - * @param connectionId Connection identifier to get the MongoDB client + * @param connectionId Connection identifier to get the DocumentDB client * @param databaseName Name of the target database * @param collectionName Name of the target collection * @returns Promise that resolves when the collection is ready @@ -180,6 +180,10 @@ export class MongoDocumentWriter implements DocumentWriter { const collections = await client.listCollections(databaseName); const collectionExists = collections.some((col) => col.name === collectionName); + // we could have just run 'createCollection' without this check. This will work just fine + // for basic scenarios. However, an exiting collection with the same name but a different + // configuration could lead to unexpected behavior. + if (!collectionExists) { // Create the collection by running createCollection await client.createCollection(databaseName, collectionName); diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts index 5e4358c49..36f4b96fa 100644 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -64,11 +64,7 @@ export class CopyPasteCollectionTask extends Task { */ protected async onInitialize(signal: AbortSignal): Promise { // Count total documents for progress calculation - this.updateStatus(this.getStatus().state, vscode.l10n.t('Counting documents in source collection...'), 0); - - if (signal.aborted) { - return; - } + this.updateStatus(this.getStatus().state, vscode.l10n.t('Counting documents in source collection...')); try { this.totalDocuments = await this.documentReader.countDocuments( @@ -90,7 +86,7 @@ export class CopyPasteCollectionTask extends Task { } // Ensure target collection exists - this.updateStatus(this.getStatus().state, vscode.l10n.t('Ensuring target collection exists...'), 0); + this.updateStatus(this.getStatus().state, vscode.l10n.t('Ensuring target collection exists...')); try { await this.documentWriter.ensureCollectionExists( @@ -116,12 +112,9 @@ export class CopyPasteCollectionTask extends Task { protected async doWork(signal: AbortSignal): Promise { // Handle the case where there are no documents to copy if (this.totalDocuments === 0) { - this.updateProgress(100, vscode.l10n.t('No documents to copy. Operation completed.')); return; } - this.updateProgress(0, vscode.l10n.t('Starting document copy...')); - const documentStream = this.documentReader.streamDocuments( this.config.source.connectionId, this.config.source.databaseName, @@ -134,10 +127,8 @@ export class CopyPasteCollectionTask extends Task { try { for await (const document of documentStream) { if (signal.aborted) { - // Cleanup any remaining buffer - if (buffer.length > 0) { - await this.flushBuffer(buffer); - } + buffer.length = 0; // Clear buffer + bufferMemoryEstimate = 0; return; } @@ -153,9 +144,19 @@ export class CopyPasteCollectionTask extends Task { } } + if (signal.aborted) { + buffer.length = 0; // Clear buffer + bufferMemoryEstimate = 0; + return; + } + // Flush any remaining documents in the buffer if (buffer.length > 0) { await this.flushBuffer(buffer); + + // not needed here, but left in place for consistency and for future maintainers. + buffer.length = 0; // Clear buffer + bufferMemoryEstimate = 0; } // Ensure we report 100% completion diff --git a/src/utils/copyPasteUtils.ts b/src/utils/copyPasteUtils.ts index 13132c930..a5b38892a 100644 --- a/src/utils/copyPasteUtils.ts +++ b/src/utils/copyPasteUtils.ts @@ -53,10 +53,10 @@ export interface CopyPasteConfig { /** * Optional reference to a connection manager or client object. * For now, this is typed as `unknown` to allow flexibility. - * Specific task implementations (e.g., for MongoDB) will cast this to their + * Specific task implementations (e.g., for DocumentDB) will cast this to their * required client/connection type. */ - connectionManager?: unknown; // e.g. could be cast to a MongoDB client instance + connectionManager?: unknown; // e.g. could be cast to a DocumentDB client instance } /** @@ -64,14 +64,14 @@ export interface CopyPasteConfig { */ export interface DocumentDetails { /** - * The document's unique identifier (e.g., _id in MongoDB) + * The document's unique identifier (e.g., _id in DocumentDB) */ id: unknown; /** * The document content treated as opaque data by the core task logic. * Specific readers/writers will know how to interpret/serialize this. - * For MongoDB, this would typically be a BSON document. + * For DocumentDB, this would typically be a BSON document. */ documentContent: unknown; } @@ -124,7 +124,6 @@ export interface BulkWriteResult { * Array of errors that occurred during the write operation. */ errors: Array<{ documentId?: string; error: Error }> | null; // Should be typed more specifically based on the implementation - // e.g., for MongoDB, this could be an array of MongoBulkWriteError objects } /** From 21b409bbf0fa0ae6d9fb4743c6b1502d68adb873 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 13:55:00 +0200 Subject: [PATCH 13/21] l10n updates --- l10n/bundle.l10n.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index cc24a7acd..d3f49db30 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -17,10 +17,15 @@ "⏩ Run All": "⏩ Run All", "⏳ Running All…": "⏳ Running All…", "⏳ Running Command…": "⏳ Running Command…", + "⏹️ Task '{taskName}' was stopped. {message}": "⏹️ Task '{taskName}' was stopped. {message}", "▶️ Run Command": "▶️ Run Command", + "▶️ Task '{taskName}' starting...": "▶️ Task '{taskName}' starting...", "⚠️ **Security:** TLS/SSL Disabled": "⚠️ **Security:** TLS/SSL Disabled", + "⚠️ Task '{taskName}' failed. {message}": "⚠️ Task '{taskName}' failed. {message}", + "⚠️ Task failed. {errorMessage}": "⚠️ Task failed. {errorMessage}", "⚠ TLS/SSL Disabled": "⚠ TLS/SSL Disabled", "✅ **Security:** TLS/SSL Enabled": "✅ **Security:** TLS/SSL Enabled", + "✅ Task '{taskName}' completed successfully. {message}": "✅ Task '{taskName}' completed successfully. {message}", "$(add) Create...": "$(add) Create...", "$(check) Success": "$(check) Success", "$(error) Failure": "$(error) Failure", @@ -83,7 +88,6 @@ "Click here to update credentials": "Click here to update credentials", "Click to view resource": "Click to view resource", "Collection \"{0}\" from database \"{1}\" has been marked for copy.": "Collection \"{0}\" from database \"{1}\" has been marked for copy.", - "Collection copied successfully: {0}": "Collection copied successfully: {0}", "Collection name cannot begin with the system. prefix (Reserved for internal use).": "Collection name cannot begin with the system. prefix (Reserved for internal use).", "Collection name cannot contain .system.": "Collection name cannot contain .system.", "Collection name cannot contain the $.": "Collection name cannot contain the $.", @@ -107,7 +111,6 @@ "Copy collection \"{0}\" from \"{1}\" to \"{2}\"": "Copy collection \"{0}\" from \"{1}\" to \"{2}\"", "Copy operation completed successfully": "Copy operation completed successfully", "Copy operation failed: {0}": "Copy operation failed: {0}", - "Copy operation was cancelled.": "Copy operation was cancelled.", "CosmosDB Accounts": "CosmosDB Accounts", "Could not find {0}": "Could not find {0}", "Could not find the Azure Resource Groups extension": "Could not find the Azure Resource Groups extension", @@ -321,7 +324,6 @@ "No commands found in this document.": "No commands found in this document.", "No Connectivity": "No Connectivity", "No credentials found for id {credentialId}": "No credentials found for id {credentialId}", - "No documents to copy. Operation completed.": "No documents to copy. Operation completed.", "No matching resources found.": "No matching resources found.", "No node selected.": "No node selected.", "No properties found in the schema at path \"{0}\"": "No properties found in the schema at path \"{0}\"", @@ -402,7 +404,6 @@ "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "Specified character lengths should be 1 character or greater.": "Specified character lengths should be 1 character or greater.", "Started executable: \"{command}\". Connecting to host…": "Started executable: \"{command}\". Connecting to host…", - "Starting document copy...": "Starting document copy...", "Starting executable: \"{command}\"": "Starting executable: \"{command}\"", "Starts with mongodb:// or mongodb+srv://": "Starts with mongodb:// or mongodb+srv://", "Stopping {0}": "Stopping {0}", From 96a841c7cb9cd1a0acb56c35a6492e7f4c333d90 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:07:27 +0200 Subject: [PATCH 14/21] feat: centralized task status udpates in the outputChannel --- l10n/bundle.l10n.json | 2 +- .../pasteCollection/pasteCollection.ts | 71 ++++++++++--------- src/services/taskService.ts | 35 +++++++++ 3 files changed, 73 insertions(+), 35 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index d3f49db30..8932614df 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -22,7 +22,6 @@ "▶️ Task '{taskName}' starting...": "▶️ Task '{taskName}' starting...", "⚠️ **Security:** TLS/SSL Disabled": "⚠️ **Security:** TLS/SSL Disabled", "⚠️ Task '{taskName}' failed. {message}": "⚠️ Task '{taskName}' failed. {message}", - "⚠️ Task failed. {errorMessage}": "⚠️ Task failed. {errorMessage}", "⚠ TLS/SSL Disabled": "⚠ TLS/SSL Disabled", "✅ **Security:** TLS/SSL Enabled": "✅ **Security:** TLS/SSL Enabled", "✅ Task '{taskName}' completed successfully. {message}": "✅ Task '{taskName}' completed successfully. {message}", @@ -418,6 +417,7 @@ "Tag cannot be empty.": "Tag cannot be empty.", "Tag cannot be longer than 256 characters.": "Tag cannot be longer than 256 characters.", "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", + "Task '{taskName}' initializing...": "Task '{taskName}' initializing...", "Task aborted because of error: {0}, {1} document(s) were inserted in total": "Task aborted because of error: {0}, {1} document(s) were inserted in total", "Task completed successfully": "Task completed successfully", "Task created and ready to start": "Task created and ready to start", diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 8cf1a65a1..ab94f8a31 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -9,7 +9,7 @@ import * as vscode from 'vscode'; import { DocumentDbDocumentReader, DocumentDbDocumentWriter } from '../../documentdb/DocumentProvider'; import { ext } from '../../extensionVariables'; import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; -import { TaskService, TaskState } from '../../services/taskService'; +import { TaskService } from '../../services/taskService'; import { CollectionItem } from '../../tree/documentdb/CollectionItem'; import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../utils/copyPasteUtils'; @@ -97,45 +97,48 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll // Register task with the task service TaskService.registerTask(task); - task.onDidChangeState((event) => { - if (event.newState === TaskState.Completed) { - const summary = task.getStatus(); - ext.outputChannel.appendLine( - l10n.t("✅ Task '{taskName}' completed successfully. {message}", { - taskName: task.name, - message: summary.message || '', - }), - ); - } else if (event.newState === TaskState.Stopped) { - ext.outputChannel.appendLine( - l10n.t("⏹️ Task '{taskName}' was stopped. {message}", { - taskName: task.name, - message: task.getStatus().message || '', - }), - ); - } else if (event.newState === TaskState.Failed) { - const summary = task.getStatus(); - ext.outputChannel.appendLine( - l10n.t("⚠️ Task '{taskName}' failed. {message}", { - taskName: task.name, - message: summary.message || '', - }), - ); - } - }); - - ext.outputChannel.appendLine(l10n.t("▶️ Task '{taskName}' starting...", { taskName: 'Copy Collection' })); + // Remove manual logging; now handled by Task base class + // task.onDidChangeState((event) => { + // if (event.newState === TaskState.Completed) { + // const summary = task.getStatus(); + // ext.outputChannel.appendLine( + // l10n.t("✅ Task '{taskName}' completed successfully. {message}", { + // taskName: task.name, + // message: summary.message || '', + // }), + // ); + // } else if (event.newState === TaskState.Stopped) { + // ext.outputChannel.appendLine( + // l10n.t("⏹️ Task '{taskName}' was stopped. {message}", { + // taskName: task.name, + // message: task.getStatus().message || '', + // }), + // ); + // } else if (event.newState === TaskState.Failed) { + // const summary = task.getStatus(); + // ext.outputChannel.appendLine( + // l10n.t("⚠️ Task '{taskName}' failed. {message}", { + // taskName: task.name, + // message: summary.message || '', + // }), + // ); + // } + // }); + + // ext.outputChannel.appendLine(l10n.t("▶️ Task '{taskName}' starting...", { taskName: 'Copy Collection' })); // Start the copy-paste task await task.start(); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); - ext.outputChannel.appendLine( - l10n.t('⚠️ Task failed. {errorMessage}', { - errorMessage: errorMessage, - }), - ); + + // Remove duplicate output log; Task base class logs failures centrally + // ext.outputChannel.appendLine( + // l10n.t('⚠️ Task failed. {errorMessage}', { + // errorMessage: errorMessage, + // }), + // ); throw error; } diff --git a/src/services/taskService.ts b/src/services/taskService.ts index 4b2f7444e..546f1dc92 100644 --- a/src/services/taskService.ts +++ b/src/services/taskService.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as vscode from 'vscode'; +import { ext } from '../extensionVariables'; /** * Enumeration of possible states a task can be in. @@ -166,6 +167,36 @@ export abstract class Task { newState: state, taskId: this.id, }); + + // Centralized logging for final state transitions + if (state === TaskState.Completed) { + const msg = this._status.message ?? ''; + ext.outputChannel.appendLine( + vscode.l10n.t("✅ Task '{taskName}' completed successfully. {message}", { + taskName: this.name, + message: msg, + }), + ); + } else if (state === TaskState.Stopped) { + const msg = this._status.message ?? ''; + ext.outputChannel.appendLine( + vscode.l10n.t("⏹️ Task '{taskName}' was stopped. {message}", { + taskName: this.name, + message: msg, + }), + ); + } else if (state === TaskState.Failed) { + const msg = this._status.message ?? ''; + const err = this._status.error instanceof Error ? this._status.error.message : ''; + // Include error details if available + const detail = err ? ` ${vscode.l10n.t('Error: {0}', err)}` : ''; + ext.outputChannel.appendLine( + vscode.l10n.t("⚠️ Task '{taskName}' failed. {message}", { + taskName: this.name, + message: `${msg}${detail}`.trim(), + }), + ); + } } } @@ -197,6 +228,9 @@ export abstract class Task { if (this._status.state !== TaskState.Pending) { throw new Error(vscode.l10n.t('Cannot start task in state: {0}', this._status.state)); } + + ext.outputChannel.appendLine(vscode.l10n.t("Task '{taskName}' initializing...", { taskName: this.name })); + this.updateStatus(TaskState.Initializing, vscode.l10n.t('Initializing task...'), 0); try { @@ -214,6 +248,7 @@ export abstract class Task { } this.updateStatus(TaskState.Running, vscode.l10n.t('Task is running'), 0); + ext.outputChannel.appendLine(vscode.l10n.t("▶️ Task '{taskName}' starting...", { taskName: this.name })); // Start the actual work asynchronously void this.runWork().catch((error) => { From a955b244724886c673213b2a4d7d8136d2d1d0ca Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:13:17 +0200 Subject: [PATCH 15/21] refreshed Copy-Paste task implememtation for readability --- l10n/bundle.l10n.json | 6 +- src/services/tasks/CopyPasteCollectionTask.ts | 101 ++++++++---------- 2 files changed, 49 insertions(+), 58 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index 8932614df..20b36800a 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -50,6 +50,7 @@ "Always upload": "Always upload", "An element with the following id already exists: {id}": "An element with the following id already exists: {id}", "An error has occurred. Check output window for more details.": "An error has occurred. Check output window for more details.", + "An error occurred while writing documents: {0}": "An error occurred while writing documents: {0}", "An item with id \"{0}\" already exists for workspace \"{1}\".": "An item with id \"{0}\" already exists for workspace \"{1}\".", "An unknown error occurred while inserting documents.": "An unknown error occurred while inserting documents.", "API version \"{0}\" for extension id \"{1}\" is no longer supported. Minimum version is \"{2}\".": "API version \"{0}\" for extension id \"{1}\" is no longer supported. Minimum version is \"{2}\".", @@ -109,7 +110,6 @@ "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.": "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.", "Copy collection \"{0}\" from \"{1}\" to \"{2}\"": "Copy collection \"{0}\" from \"{1}\" to \"{2}\"", "Copy operation completed successfully": "Copy operation completed successfully", - "Copy operation failed: {0}": "Copy operation failed: {0}", "CosmosDB Accounts": "CosmosDB Accounts", "Could not find {0}": "Could not find {0}", "Could not find the Azure Resource Groups extension": "Could not find the Azure Resource Groups extension", @@ -400,6 +400,7 @@ "Skipped document with _id: {0} due to error: {1}": "Skipped document with _id: {0} due to error: {1}", "Small breadcrumb example with buttons": "Small breadcrumb example with buttons", "Some items could not be displayed": "Some items could not be displayed", + "Source collection is empty.": "Source collection is empty.", "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Source: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "Specified character lengths should be 1 character or greater.": "Specified character lengths should be 1 character or greater.", "Started executable: \"{command}\". Connecting to host…": "Started executable: \"{command}\". Connecting to host…", @@ -418,11 +419,10 @@ "Tag cannot be longer than 256 characters.": "Tag cannot be longer than 256 characters.", "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", "Task '{taskName}' initializing...": "Task '{taskName}' initializing...", - "Task aborted because of error: {0}, {1} document(s) were inserted in total": "Task aborted because of error: {0}, {1} document(s) were inserted in total", + "Task aborted due to an error: {0}. {1} document(s) were inserted in total.": "Task aborted due to an error: {0}. {1} document(s) were inserted in total.", "Task completed successfully": "Task completed successfully", "Task created and ready to start": "Task created and ready to start", "Task failed": "Task failed", - "Task failed due to error: {0}": "Task failed due to error: {0}", "Task is running": "Task is running", "Task stopped": "Task stopped", "Task stopped during initialization": "Task stopped during initialization", diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/CopyPasteCollectionTask.ts index 36f4b96fa..cc9789109 100644 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ b/src/services/tasks/CopyPasteCollectionTask.ts @@ -15,11 +15,11 @@ import { import { Task } from '../taskService'; /** - * Task for copying documents from a source collection to a target collection. + * Task for copying documents from a source to a target collection. * - * This task implements a database-agnostic approach using DocumentReader and DocumentWriter - * interfaces to handle the actual data operations. It manages memory efficiently through - * a buffer-based streaming approach where documents are read and written in batches. + * This task uses a database-agnostic approach with `DocumentReader` and `DocumentWriter` + * interfaces. It streams documents from the source and writes them in batches to the + * target, managing memory usage with a configurable buffer. */ export class CopyPasteCollectionTask extends Task { public readonly type: string = 'copy-paste-collection'; @@ -112,6 +112,7 @@ export class CopyPasteCollectionTask extends Task { protected async doWork(signal: AbortSignal): Promise { // Handle the case where there are no documents to copy if (this.totalDocuments === 0) { + this.updateProgress(100, vscode.l10n.t('Source collection is empty.')); return; } @@ -124,62 +125,45 @@ export class CopyPasteCollectionTask extends Task { const buffer: DocumentDetails[] = []; let bufferMemoryEstimate = 0; - try { - for await (const document of documentStream) { - if (signal.aborted) { - buffer.length = 0; // Clear buffer - bufferMemoryEstimate = 0; - return; - } - - // Add document to buffer - buffer.push(document); - bufferMemoryEstimate += this.estimateDocumentMemory(document); - - // Check if we need to flush the buffer - if (this.shouldFlushBuffer(buffer.length, bufferMemoryEstimate)) { - await this.flushBuffer(buffer); - buffer.length = 0; // Clear buffer - bufferMemoryEstimate = 0; - } - } - + for await (const document of documentStream) { if (signal.aborted) { - buffer.length = 0; // Clear buffer - bufferMemoryEstimate = 0; + // Buffer is a local variable, no need to clear, just exit. return; } - // Flush any remaining documents in the buffer - if (buffer.length > 0) { - await this.flushBuffer(buffer); + // Add document to buffer + buffer.push(document); + bufferMemoryEstimate += this.estimateDocumentMemory(document); - // not needed here, but left in place for consistency and for future maintainers. + // Check if we need to flush the buffer + if (this.shouldFlushBuffer(buffer.length, bufferMemoryEstimate)) { + await this.flushBuffer(buffer, signal); buffer.length = 0; // Clear buffer bufferMemoryEstimate = 0; } + } - // Ensure we report 100% completion - this.updateProgress(100, vscode.l10n.t('Copy operation completed successfully')); - } catch (error) { - // For basic implementation, any error should abort the operation - if (this.config.onConflict === ConflictResolutionStrategy.Abort) { - throw new Error( - vscode.l10n.t('Copy operation failed: {0}', error instanceof Error ? error.message : String(error)), - ); - } - // Future: Handle other conflict resolution strategies - throw error; + if (signal.aborted) { + return; } + + // Flush any remaining documents in the buffer + if (buffer.length > 0) { + await this.flushBuffer(buffer, signal); + } + + // Ensure we report 100% completion + this.updateProgress(100, vscode.l10n.t('Copy operation completed successfully')); } /** * Flushes the document buffer by writing all documents to the target collection. * - * @param buffer Array of documents to write + * @param buffer Array of documents to write. + * @param signal AbortSignal to check for cancellation. */ - private async flushBuffer(buffer: DocumentDetails[]): Promise { - if (buffer.length === 0) { + private async flushBuffer(buffer: DocumentDetails[], signal: AbortSignal): Promise { + if (buffer.length === 0 || signal.aborted) { return; } @@ -196,19 +180,20 @@ export class CopyPasteCollectionTask extends Task { this.processedDocuments += result.insertedCount; // Check for errors in the write result - if (result.errors !== null) { - // For basic implementation with abort strategy, any error should fail the task + if (result.errors && result.errors.length > 0) { + // Handle errors based on the configured conflict resolution strategy. if (this.config.onConflict === ConflictResolutionStrategy.Abort) { + // Abort strategy: fail the entire task on the first error. const firstError = result.errors[0] as { error: Error }; throw new Error( vscode.l10n.t( - 'Task aborted because of error: {0}, {1} document(s) were inserted in total', + 'Task aborted due to an error: {0}. {1} document(s) were inserted in total.', firstError.error?.message ?? 'Unknown error', this.processedDocuments.toString(), ), ); } else if (this.config.onConflict === ConflictResolutionStrategy.Skip) { - // For skip strategy, we can log the errors but continue + // Skip strategy: log each error and continue. for (const error of result.errors) { ext.outputChannel.appendLog( vscode.l10n.t( @@ -220,10 +205,15 @@ export class CopyPasteCollectionTask extends Task { } ext.outputChannel.show(); } else { - ext.outputChannel.appendLog( - vscode.l10n.t('Task failed due to error: {0}', result.errors[0].error?.message ?? 'Unknown error'), + // Overwrite or other strategies: treat errors as fatal for now. + // This can be expanded if other strategies need more nuanced error handling. + const firstError = result.errors[0] as { error: Error }; + throw new Error( + vscode.l10n.t( + 'An error occurred while writing documents: {0}', + firstError.error?.message ?? 'Unknown error', + ), ); - ext.outputChannel.show(); } } @@ -266,12 +256,13 @@ export class CopyPasteCollectionTask extends Task { */ private estimateDocumentMemory(document: DocumentDetails): number { try { - // Rough estimate: JSON stringify the document content + // A rough estimate based on the length of the JSON string representation. + // V8 strings are typically 2 bytes per character (UTF-16). const jsonString = JSON.stringify(document.documentContent); - return jsonString.length * 2; // Rough estimate for UTF-16 encoding + return jsonString.length * 2; } catch { - // If we can't serialize, use a conservative estimate - return 1024; // 1KB default estimate + // If serialization fails, return a conservative default. + return 1024; // 1KB } } } From 7582e3b9f844d3476bd5406bb4649442a5217aed Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:15:41 +0200 Subject: [PATCH 16/21] ux tweak / emojii... --- l10n/bundle.l10n.json | 2 +- src/services/taskService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index 20b36800a..e923d0909 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -25,6 +25,7 @@ "⚠ TLS/SSL Disabled": "⚠ TLS/SSL Disabled", "✅ **Security:** TLS/SSL Enabled": "✅ **Security:** TLS/SSL Enabled", "✅ Task '{taskName}' completed successfully. {message}": "✅ Task '{taskName}' completed successfully. {message}", + "🟡 Task '{taskName}' initializing...": "🟡 Task '{taskName}' initializing...", "$(add) Create...": "$(add) Create...", "$(check) Success": "$(check) Success", "$(error) Failure": "$(error) Failure", @@ -418,7 +419,6 @@ "Tag cannot be empty.": "Tag cannot be empty.", "Tag cannot be longer than 256 characters.": "Tag cannot be longer than 256 characters.", "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}": "Target: Collection \"{0}\" from database \"{1}\", connectionId: {2}", - "Task '{taskName}' initializing...": "Task '{taskName}' initializing...", "Task aborted due to an error: {0}. {1} document(s) were inserted in total.": "Task aborted due to an error: {0}. {1} document(s) were inserted in total.", "Task completed successfully": "Task completed successfully", "Task created and ready to start": "Task created and ready to start", diff --git a/src/services/taskService.ts b/src/services/taskService.ts index 546f1dc92..7efd36ec3 100644 --- a/src/services/taskService.ts +++ b/src/services/taskService.ts @@ -229,7 +229,7 @@ export abstract class Task { throw new Error(vscode.l10n.t('Cannot start task in state: {0}', this._status.state)); } - ext.outputChannel.appendLine(vscode.l10n.t("Task '{taskName}' initializing...", { taskName: this.name })); + ext.outputChannel.appendLine(vscode.l10n.t("🟡 Task '{taskName}' initializing...", { taskName: this.name })); this.updateStatus(TaskState.Initializing, vscode.l10n.t('Initializing task...'), 0); From 3037998cceb7e1644dd1eddd58c45df114a800ca Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 12:24:43 +0000 Subject: [PATCH 17/21] refactoring file locations --- src/commands/pasteCollection/pasteCollection.ts | 6 +++--- .../tasks/{ => copy-and-paste}/CopyPasteCollectionTask.ts | 6 +++--- .../tasks/copy-and-paste}/DocumentProvider.ts | 4 ++-- .../tasks/copy-and-paste}/copyPasteUtils.ts | 0 4 files changed, 8 insertions(+), 8 deletions(-) rename src/services/tasks/{ => copy-and-paste}/CopyPasteCollectionTask.ts (98%) rename src/{documentdb => services/tasks/copy-and-paste}/DocumentProvider.ts (98%) rename src/{utils => services/tasks/copy-and-paste}/copyPasteUtils.ts (100%) diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index ab94f8a31..6ba6b336c 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -6,12 +6,12 @@ import { type IActionContext } from '@microsoft/vscode-azext-utils'; import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; -import { DocumentDbDocumentReader, DocumentDbDocumentWriter } from '../../documentdb/DocumentProvider'; +import { DocumentDbDocumentReader, DocumentDbDocumentWriter } from '../../services/tasks/copy-and-paste/DocumentProvider'; import { ext } from '../../extensionVariables'; -import { CopyPasteCollectionTask } from '../../services/tasks/CopyPasteCollectionTask'; +import { CopyPasteCollectionTask } from '../../services/tasks/copy-and-paste/CopyPasteCollectionTask'; import { TaskService } from '../../services/taskService'; import { CollectionItem } from '../../tree/documentdb/CollectionItem'; -import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../utils/copyPasteUtils'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../services/tasks/copy-and-paste/copyPasteUtils'; export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { const sourceNode = ext.copiedCollectionNode; diff --git a/src/services/tasks/CopyPasteCollectionTask.ts b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts similarity index 98% rename from src/services/tasks/CopyPasteCollectionTask.ts rename to src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts index cc9789109..d67235c77 100644 --- a/src/services/tasks/CopyPasteCollectionTask.ts +++ b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts @@ -4,15 +4,15 @@ *--------------------------------------------------------------------------------------------*/ import * as vscode from 'vscode'; -import { ext } from '../../extensionVariables'; +import { ext } from '../../../extensionVariables'; import { type CopyPasteConfig, type DocumentDetails, type DocumentReader, type DocumentWriter, ConflictResolutionStrategy, -} from '../../utils/copyPasteUtils'; -import { Task } from '../taskService'; +} from './copyPasteUtils'; +import { Task } from '../../taskService'; /** * Task for copying documents from a source to a target collection. diff --git a/src/documentdb/DocumentProvider.ts b/src/services/tasks/copy-and-paste/DocumentProvider.ts similarity index 98% rename from src/documentdb/DocumentProvider.ts rename to src/services/tasks/copy-and-paste/DocumentProvider.ts index b08444133..d35663b15 100644 --- a/src/documentdb/DocumentProvider.ts +++ b/src/services/tasks/copy-and-paste/DocumentProvider.ts @@ -6,7 +6,7 @@ import { parseError } from '@microsoft/vscode-azext-utils'; import { type Document, type ObjectId, type WithId, type WriteError } from 'mongodb'; import { l10n } from 'vscode'; -import { ClustersClient, isBulkWriteError } from '../documentdb/ClustersClient'; +import { ClustersClient, isBulkWriteError } from '../../../documentdb/ClustersClient'; import { ConflictResolutionStrategy, type BulkWriteResult, @@ -15,7 +15,7 @@ import { type DocumentReader, type DocumentWriter, type DocumentWriterOptions, -} from '../utils/copyPasteUtils'; +} from './copyPasteUtils'; /** * DocumentDB-specific implementation of DocumentReader. diff --git a/src/utils/copyPasteUtils.ts b/src/services/tasks/copy-and-paste/copyPasteUtils.ts similarity index 100% rename from src/utils/copyPasteUtils.ts rename to src/services/tasks/copy-and-paste/copyPasteUtils.ts From cd7983d19b66baf60b6a27ad1cb23b0f56c9aea0 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:43:08 +0200 Subject: [PATCH 18/21] finalized refactoring --- .../pasteCollection/pasteCollection.ts | 5 +- .../copy-and-paste/CopyPasteCollectionTask.ts | 9 +-- .../tasks/copy-and-paste/copyPasteConfig.ts | 60 +++++++++++++++++++ ...opyPasteUtils.ts => documentInterfaces.ts} | 56 +---------------- .../documentdb/documentDbDocumentReader.ts | 56 +++++++++++++++++ .../documentDbDocumentWriter.ts} | 56 +---------------- 6 files changed, 125 insertions(+), 117 deletions(-) create mode 100644 src/services/tasks/copy-and-paste/copyPasteConfig.ts rename src/services/tasks/copy-and-paste/{copyPasteUtils.ts => documentInterfaces.ts} (72%) create mode 100644 src/services/tasks/copy-and-paste/documentdb/documentDbDocumentReader.ts rename src/services/tasks/copy-and-paste/{DocumentProvider.ts => documentdb/documentDbDocumentWriter.ts} (76%) diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 6ba6b336c..50d6c790f 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -6,12 +6,13 @@ import { type IActionContext } from '@microsoft/vscode-azext-utils'; import * as l10n from '@vscode/l10n'; import * as vscode from 'vscode'; -import { DocumentDbDocumentReader, DocumentDbDocumentWriter } from '../../services/tasks/copy-and-paste/DocumentProvider'; import { ext } from '../../extensionVariables'; import { CopyPasteCollectionTask } from '../../services/tasks/copy-and-paste/CopyPasteCollectionTask'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../services/tasks/copy-and-paste/copyPasteConfig'; +import { DocumentDbDocumentReader } from '../../services/tasks/copy-and-paste/documentdb/documentDbDocumentReader'; +import { DocumentDbDocumentWriter } from '../../services/tasks/copy-and-paste/documentdb/documentDbDocumentWriter'; import { TaskService } from '../../services/taskService'; import { CollectionItem } from '../../tree/documentdb/CollectionItem'; -import { ConflictResolutionStrategy, type CopyPasteConfig } from '../../services/tasks/copy-and-paste/copyPasteUtils'; export async function pasteCollection(_context: IActionContext, targetNode: CollectionItem): Promise { const sourceNode = ext.copiedCollectionNode; diff --git a/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts index d67235c77..3e218fec2 100644 --- a/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts +++ b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts @@ -5,14 +5,9 @@ import * as vscode from 'vscode'; import { ext } from '../../../extensionVariables'; -import { - type CopyPasteConfig, - type DocumentDetails, - type DocumentReader, - type DocumentWriter, - ConflictResolutionStrategy, -} from './copyPasteUtils'; import { Task } from '../../taskService'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from './copyPasteConfig'; +import { type DocumentDetails, type DocumentReader, type DocumentWriter } from './documentInterfaces'; /** * Task for copying documents from a source to a target collection. diff --git a/src/services/tasks/copy-and-paste/copyPasteConfig.ts b/src/services/tasks/copy-and-paste/copyPasteConfig.ts new file mode 100644 index 000000000..a787c49e8 --- /dev/null +++ b/src/services/tasks/copy-and-paste/copyPasteConfig.ts @@ -0,0 +1,60 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Enumeration of conflict resolution strategies for copy-paste operations + */ +export enum ConflictResolutionStrategy { + /** + * Abort the operation if any conflict or error occurs + */ + Abort = 'abort', + + /** + * Skip the conflicting document and continue with the operation + */ + Skip = 'skip', + + /** + * Overwrite the existing document in the target collection with the source document + */ + Overwrite = 'overwrite', +} + +/** + * Configuration for copy-paste operations + */ +export interface CopyPasteConfig { + /** + * Source collection information + */ + source: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Target collection information + */ + target: { + connectionId: string; + databaseName: string; + collectionName: string; + }; + + /** + * Conflict resolution strategy + */ + onConflict: ConflictResolutionStrategy; + + /** + * Optional reference to a connection manager or client object. + * For now, this is typed as `unknown` to allow flexibility. + * Specific task implementations (e.g., for DocumentDB) will cast this to their + * required client/connection type. + */ + connectionManager?: unknown; // e.g. could be cast to a DocumentDB client instance +} diff --git a/src/services/tasks/copy-and-paste/copyPasteUtils.ts b/src/services/tasks/copy-and-paste/documentInterfaces.ts similarity index 72% rename from src/services/tasks/copy-and-paste/copyPasteUtils.ts rename to src/services/tasks/copy-and-paste/documentInterfaces.ts index a5b38892a..82c90a8fa 100644 --- a/src/services/tasks/copy-and-paste/copyPasteUtils.ts +++ b/src/services/tasks/copy-and-paste/documentInterfaces.ts @@ -3,61 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -/** - * Enumeration of conflict resolution strategies for copy-paste operations - */ -export enum ConflictResolutionStrategy { - /** - * Abort the operation if any conflict or error occurs - */ - Abort = 'abort', - - /** - * Skip the conflicting document and continue with the operation - */ - Skip = 'skip', - - /** - * Overwrite the existing document in the target collection with the source document - */ - Overwrite = 'overwrite', -} - -/** - * Configuration for copy-paste operations - */ -export interface CopyPasteConfig { - /** - * Source collection information - */ - source: { - connectionId: string; - databaseName: string; - collectionName: string; - }; - - /** - * Target collection information - */ - target: { - connectionId: string; - databaseName: string; - collectionName: string; - }; - - /** - * Conflict resolution strategy - */ - onConflict: ConflictResolutionStrategy; - - /** - * Optional reference to a connection manager or client object. - * For now, this is typed as `unknown` to allow flexibility. - * Specific task implementations (e.g., for DocumentDB) will cast this to their - * required client/connection type. - */ - connectionManager?: unknown; // e.g. could be cast to a DocumentDB client instance -} +import { type CopyPasteConfig } from './copyPasteConfig'; /** * Represents a single document in the copy-paste operation. diff --git a/src/services/tasks/copy-and-paste/documentdb/documentDbDocumentReader.ts b/src/services/tasks/copy-and-paste/documentdb/documentDbDocumentReader.ts new file mode 100644 index 000000000..523228094 --- /dev/null +++ b/src/services/tasks/copy-and-paste/documentdb/documentDbDocumentReader.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type Document, type WithId } from 'mongodb'; +import { ClustersClient } from '../../../../documentdb/ClustersClient'; +import { type DocumentDetails, type DocumentReader } from '../documentInterfaces'; + +/** + * DocumentDB-specific implementation of DocumentReader. + */ +export class DocumentDbDocumentReader implements DocumentReader { + /** + * Streams documents from a DocumentDB collection. + * + * @param connectionId Connection identifier to get the DocumentDB client + * @param databaseName Name of the database + * @param collectionName Name of the collection + * @returns AsyncIterable of document details + */ + async *streamDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + ): AsyncIterable { + const client = await ClustersClient.getClient(connectionId); + + const documentStream = client.streamDocuments(databaseName, collectionName, new AbortController().signal); + for await (const document of documentStream) { + yield { + id: (document as WithId)._id, + documentContent: document, + }; + } + } + + /** + * Counts the total number of documents in the DocumentDB collection. + * + * @param connectionId Connection identifier to get the DocumentDB client + * @param databaseName Name of the database + * @param collectionName Name of the collection, + * @param filter Optional filter to apply to the count operation (default is '{}') + * @returns Promise resolving to the document count + */ + async countDocuments( + connectionId: string, + databaseName: string, + collectionName: string, + filter: string = '{}', + ): Promise { + const client = await ClustersClient.getClient(connectionId); + return await client.countDocuments(databaseName, collectionName, filter); + } +} diff --git a/src/services/tasks/copy-and-paste/DocumentProvider.ts b/src/services/tasks/copy-and-paste/documentdb/documentDbDocumentWriter.ts similarity index 76% rename from src/services/tasks/copy-and-paste/DocumentProvider.ts rename to src/services/tasks/copy-and-paste/documentdb/documentDbDocumentWriter.ts index d35663b15..eb08a3184 100644 --- a/src/services/tasks/copy-and-paste/DocumentProvider.ts +++ b/src/services/tasks/copy-and-paste/documentdb/documentDbDocumentWriter.ts @@ -6,64 +6,14 @@ import { parseError } from '@microsoft/vscode-azext-utils'; import { type Document, type ObjectId, type WithId, type WriteError } from 'mongodb'; import { l10n } from 'vscode'; -import { ClustersClient, isBulkWriteError } from '../../../documentdb/ClustersClient'; +import { ClustersClient, isBulkWriteError } from '../../../../documentdb/ClustersClient'; +import { ConflictResolutionStrategy, type CopyPasteConfig } from '../copyPasteConfig'; import { - ConflictResolutionStrategy, type BulkWriteResult, - type CopyPasteConfig, type DocumentDetails, - type DocumentReader, type DocumentWriter, type DocumentWriterOptions, -} from './copyPasteUtils'; - -/** - * DocumentDB-specific implementation of DocumentReader. - */ -export class DocumentDbDocumentReader implements DocumentReader { - /** - * Streams documents from a DocumentDB collection. - * - * @param connectionId Connection identifier to get the DocumentDB client - * @param databaseName Name of the database - * @param collectionName Name of the collection - * @returns AsyncIterable of document details - */ - async *streamDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - ): AsyncIterable { - const client = await ClustersClient.getClient(connectionId); - - const documentStream = client.streamDocuments(databaseName, collectionName, new AbortController().signal); - for await (const document of documentStream) { - yield { - id: (document as WithId)._id, - documentContent: document, - }; - } - } - - /** - * Counts the total number of documents in the DocumentDB collection. - * - * @param connectionId Connection identifier to get the DocumentDB client - * @param databaseName Name of the database - * @param collectionName Name of the collection, - * @param filter Optional filter to apply to the count operation (default is '{}') - * @returns Promise resolving to the document count - */ - async countDocuments( - connectionId: string, - databaseName: string, - collectionName: string, - filter: string = '{}', - ): Promise { - const client = await ClustersClient.getClient(connectionId); - return await client.countDocuments(databaseName, collectionName, filter); - } -} +} from '../documentInterfaces'; /** * DocumentDB-specific implementation of DocumentWriter. From e5672421e9fe864ea394bf4f284cafc4dfda69ff Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:45:47 +0200 Subject: [PATCH 19/21] removed obsolete code / commented out code. --- .../pasteCollection/pasteCollection.ts | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/src/commands/pasteCollection/pasteCollection.ts b/src/commands/pasteCollection/pasteCollection.ts index 50d6c790f..d836aa4e2 100644 --- a/src/commands/pasteCollection/pasteCollection.ts +++ b/src/commands/pasteCollection/pasteCollection.ts @@ -88,59 +88,15 @@ export async function pasteCollection(_context: IActionContext, targetNode: Coll const writer = new DocumentDbDocumentWriter(); const task = new CopyPasteCollectionTask(config, reader, writer); - // // Get total number of documents in the source collection - // const totalDocuments = await reader.countDocuments( - // config.source.connectionId, - // config.source.databaseName, - // config.source.collectionName, - // ); - // Register task with the task service TaskService.registerTask(task); - // Remove manual logging; now handled by Task base class - // task.onDidChangeState((event) => { - // if (event.newState === TaskState.Completed) { - // const summary = task.getStatus(); - // ext.outputChannel.appendLine( - // l10n.t("✅ Task '{taskName}' completed successfully. {message}", { - // taskName: task.name, - // message: summary.message || '', - // }), - // ); - // } else if (event.newState === TaskState.Stopped) { - // ext.outputChannel.appendLine( - // l10n.t("⏹️ Task '{taskName}' was stopped. {message}", { - // taskName: task.name, - // message: task.getStatus().message || '', - // }), - // ); - // } else if (event.newState === TaskState.Failed) { - // const summary = task.getStatus(); - // ext.outputChannel.appendLine( - // l10n.t("⚠️ Task '{taskName}' failed. {message}", { - // taskName: task.name, - // message: summary.message || '', - // }), - // ); - // } - // }); - - // ext.outputChannel.appendLine(l10n.t("▶️ Task '{taskName}' starting...", { taskName: 'Copy Collection' })); - // Start the copy-paste task await task.start(); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); void vscode.window.showErrorMessage(l10n.t('Failed to copy collection: {0}', errorMessage)); - // Remove duplicate output log; Task base class logs failures centrally - // ext.outputChannel.appendLine( - // l10n.t('⚠️ Task failed. {errorMessage}', { - // errorMessage: errorMessage, - // }), - // ); - throw error; } } From 884116d4af965918d22039ec648e12aef768ba39 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 14:54:48 +0200 Subject: [PATCH 20/21] tweak to the copy-and-paste task display name --- l10n/bundle.l10n.json | 2 +- .../tasks/copy-and-paste/CopyPasteCollectionTask.ts | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index e923d0909..3c2957a81 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -109,7 +109,7 @@ "Copied {0} of {1} documents": "Copied {0} of {1} documents", "Copy": "Copy", "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.": "Copy \"{0}\"\nto \"{1}\"?\nThis will add all documents from the source collection to the target collection.", - "Copy collection \"{0}\" from \"{1}\" to \"{2}\"": "Copy collection \"{0}\" from \"{1}\" to \"{2}\"", + "Copy \"{sourceCollection}\" from \"{sourceDatabase}\" to \"{targetDatabase}/{targetCollection}\"": "Copy \"{sourceCollection}\" from \"{sourceDatabase}\" to \"{targetDatabase}/{targetCollection}\"", "Copy operation completed successfully": "Copy operation completed successfully", "CosmosDB Accounts": "CosmosDB Accounts", "Could not find {0}": "Could not find {0}", diff --git a/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts index 3e218fec2..cc004fd93 100644 --- a/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts +++ b/src/services/tasks/copy-and-paste/CopyPasteCollectionTask.ts @@ -45,10 +45,13 @@ export class CopyPasteCollectionTask extends Task { // Generate a descriptive name for the task this.name = vscode.l10n.t( - 'Copy collection "{0}" from "{1}" to "{2}"', - config.source.collectionName, - config.source.databaseName, - config.target.databaseName, + 'Copy "{sourceCollection}" from "{sourceDatabase}" to "{targetDatabase}/{targetCollection}"', + { + sourceCollection: config.source.collectionName, + sourceDatabase: config.source.databaseName, + targetDatabase: config.target.databaseName, + targetCollection: config.target.collectionName, + }, ); } From 85c31a48a872ab23752e860597fbbbb082b18a70 Mon Sep 17 00:00:00 2001 From: Tomasz Naumowicz Date: Wed, 13 Aug 2025 15:01:26 +0200 Subject: [PATCH 21/21] fix: updated taskService tests to ignore ext.outputChannel.appendLine calls --- src/services/taskService.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/services/taskService.test.ts b/src/services/taskService.test.ts index 112319972..0e49beab9 100644 --- a/src/services/taskService.test.ts +++ b/src/services/taskService.test.ts @@ -5,6 +5,15 @@ import { Task, TaskService, TaskState, type TaskStatus } from './taskService'; +// Mock extensionVariables (ext) module +jest.mock('../extensionVariables', () => ({ + ext: { + outputChannel: { + appendLine: jest.fn(), // Mock appendLine as a no-op function + }, + }, +})); + // Mock vscode module jest.mock('vscode', () => ({ l10n: {