diff --git a/.github/scripts/end2end/setup-e2e-env.sh b/.github/scripts/end2end/setup-e2e-env.sh index 3b67f34124..6d1079f05f 100755 --- a/.github/scripts/end2end/setup-e2e-env.sh +++ b/.github/scripts/end2end/setup-e2e-env.sh @@ -278,8 +278,6 @@ else # Azure archive settings export AZURE_ARCHIVE_ACCESS_TIER="Hot" export AZURE_ARCHIVE_MANIFEST_ACCESS_TIER="Hot" - export AZURE_BLOB_URL="${AZURE_BACKEND_ENDPOINT}" - export AZURE_QUEUE_URL="${AZURE_BACKEND_QUEUE_ENDPOINT}" # --- 15. Grant Kube API access (needed by CTST for CronJob/Pod operations) --- kubectl create clusterrolebinding serviceaccounts-cluster-admin \ diff --git a/.github/scripts/mocks/azure-mock.yaml b/.github/scripts/mocks/azure-mock.yaml index 3dc074cfe1..1db32957d4 100644 --- a/.github/scripts/mocks/azure-mock.yaml +++ b/.github/scripts/mocks/azure-mock.yaml @@ -25,7 +25,7 @@ spec: hostname: devstoreaccount1 subdomain: azure-mock containers: - - image: mcr.microsoft.com/azure-storage/azurite:3.35.0 + - image: mcr.microsoft.com/azure-storage/azurite:3.37.0 command: [ "azurite", "-l", "/data", diff --git a/tests/functional/clients/azure.ts b/tests/functional/clients/azure.ts new file mode 100644 index 0000000000..11d9c4202e --- /dev/null +++ b/tests/functional/clients/azure.ts @@ -0,0 +1,104 @@ +import { + BlobServiceClient, + StorageSharedKeyCredential, + BlobGetPropertiesResponse, + BlobItem, +} from '@azure/storage-blob'; + +import { + QueueServiceClient, + StorageSharedKeyCredential as StorageQueueSharedKeyCredential, +} from '@azure/storage-queue'; + +export type AzureCreds = { + accountName: string; + accountKey: string; +}; + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +export default class AzureClient { + private readonly blobClient: BlobServiceClient; + private readonly queueClient: QueueServiceClient; + + constructor(creds: AzureCreds) { + this.blobClient = new BlobServiceClient( + requireEnv('AZURE_BACKEND_ENDPOINT'), + new StorageSharedKeyCredential(creds.accountName, creds.accountKey), + ); + this.queueClient = new QueueServiceClient( + requireEnv('AZURE_BACKEND_QUEUE_ENDPOINT'), + new StorageQueueSharedKeyCredential(creds.accountName, creds.accountKey), + ); + } + + async listBlobs(container: string): Promise { + const blobList: BlobItem[] = []; + const iter = await this.blobClient.getContainerClient(container).listBlobsFlat(); + let blobItem = await iter.next(); + while (!blobItem.done) { + blobList.push(blobItem.value); + blobItem = await iter.next(); + } + return blobList; + } + + async getBlobProperties(container: string, blob: string): Promise { + const blobClient = this.blobClient.getContainerClient(container).getBlockBlobClient(blob); + return blobClient.getProperties(); + } + + async blobExists(container: string, blob: string): Promise { + const blobClient = this.blobClient.getContainerClient(container).getBlockBlobClient(blob); + return blobClient.exists(); + } + + async deleteBlob(container: string, blob: string): Promise { + const res = await this.blobClient + .getContainerClient(container) + .getBlockBlobClient(blob) + .deleteIfExists(); + return res.succeeded; + } + + async downloadBlob(container: string, blob: string): Promise { + return this.blobClient.getContainerClient(container).getBlockBlobClient(blob).downloadToBuffer(); + } + + async sendBlobCreatedEventToQueue(queue: string, container: string, blob: string): Promise { + const message = { + topic: '/subscriptions/0/resourceGroups/Storage/providers/Microsoft.Storage/storageAccounts/account', + subject: `/blobServices/default/containers/${container}/blobs/${blob}`, + eventType: 'Microsoft.Storage.BlobCreated', + eventTime: '2017-06-26T18:41:00.9584103Z', + id: '831e1650-001e-001b-66ab-eeb76e069631', + data: { + api: 'CopyBlob', + clientRequestId: '6d79dbfb-0e37-4fc4-981f-442c9ca65760', + requestId: '831e1650-001e-001b-66ab-eeb76e000000', + eTag: "'0x8D4BCC2E4835CD0'", + contentType: 'text/plain', + contentLength: 524288, + blobType: 'BlockBlob', + url: `https://my-storage-account.blob.core.windows.net/${container}/${blob}`, + sequencer: '00000000000004420000000000028963', + storageDiagnostics: { + batchId: 'b68529f3-68cd-4744-baa4-3c0498ec19f0', + }, + }, + dataVersion: '', + metadataVersion: '1', + }; + + const msgString = JSON.stringify(message); + const msgBuffer = Buffer.from(msgString); + + await this.queueClient.getQueueClient(queue).sendMessage(msgBuffer.toString('base64')); + } +} diff --git a/tests/functional/testResourcesSetup/setup/clients/k8s.ts b/tests/functional/clients/k8s.ts similarity index 100% rename from tests/functional/testResourcesSetup/setup/clients/k8s.ts rename to tests/functional/clients/k8s.ts diff --git a/tests/functional/testResourcesSetup/setup/clients/pensieveClient.ts b/tests/functional/clients/pensieveClient.ts similarity index 100% rename from tests/functional/testResourcesSetup/setup/clients/pensieveClient.ts rename to tests/functional/clients/pensieveClient.ts diff --git a/tests/functional/ctst/steps/azureArchive.ts b/tests/functional/ctst/steps/azureArchive.ts index 398eb595fe..ad92b2aca2 100644 --- a/tests/functional/ctst/steps/azureArchive.ts +++ b/tests/functional/ctst/steps/azureArchive.ts @@ -3,7 +3,7 @@ import path from 'path'; import assert from 'assert'; import { safeJsonParse, request } from '../common/utils'; import { Given, Then, When } from '@cucumber/cucumber'; -import { AzureHelper, S3, Constants, Utils } from 'cli-testing'; +import { S3, Constants, Utils } from 'cli-testing'; import util from 'util'; import { exec } from 'child_process'; import Zenko from 'world/Zenko'; @@ -33,19 +33,6 @@ type manifest = { 'entries': manifestEntry[], } -/** - * Returns an object containing azure credentials - * @param {Zenko} world world object - * @returns {object} azure creds - */ -function getAzureCreds( - world: Zenko, -): {accountName: string, accountKey: string } { - return { - accountName: world.parameters.AzureAccountName, - accountKey: world.parameters.AzureAccountKey, - }; -} /** * Verify that an object has well been rehydrated in azure * @param {Zenko} zenko zenko object @@ -62,10 +49,9 @@ async function isObjectRehydrated(zenko: Zenko, objectName: string) { const start = Date.now(); //wait for 1 minute max while (Date.now() - start <= 60000) { - const found = await AzureHelper.blobExists( + const found = await zenko.azureClient.blobExists( zenko.parameters.AzureArchiveContainer, `rehydrate/${tarName}`, - getAzureCreds(zenko), ); if (found) { return tarName; @@ -89,18 +75,16 @@ async function findObjectPackAndManifest( objectName: string, ): Promise<{ manifestName?:string, manifest?:manifest, tarName?:string }> { // lisintg all blobs in the container - const blobs = await AzureHelper.listBlobs( + const blobs = await world.azureClient.listBlobs( world.parameters.AzureArchiveContainer, - getAzureCreds(world), ); // filtering the list of blobs only leaving the manifests const manifests = blobs.filter(blob => blob.name.includes('.json.')); for (let i = 0; i < manifests.length; i++) { // downloading the manifest - const manifestBuffer = await AzureHelper.downloadBlob( + const manifestBuffer = await world.azureClient.downloadBlob( world.parameters.AzureArchiveContainer, manifests[i].name, - getAzureCreds(world), ); const { ok, result } = safeJsonParse(manifestBuffer.toString()); if (!ok) { @@ -168,22 +152,19 @@ export async function cleanAzureContainer( currentKey.value as string, ); if (tarName) { - await AzureHelper.deleteBlob( + await world.azureClient.deleteBlob( world.parameters.AzureArchiveContainer, tarName, - getAzureCreds(world), ); - await AzureHelper.deleteBlob( + await world.azureClient.deleteBlob( world.parameters.AzureArchiveContainer, `rehydrate/${tarName}`, - getAzureCreds(world), ); } if (manifestName) { - await AzureHelper.deleteBlob( + await world.azureClient.deleteBlob( world.parameters.AzureArchiveContainer, manifestName, - getAzureCreds(world), ); } currentKey = iterator.next(); @@ -200,10 +181,9 @@ Then('manifest access tier should be valid for object {string}', async function ); assert(manifestName); // manifest access tier - const manifestProperties = await AzureHelper.getBlobProperties( + const manifestProperties = await this.azureClient.getBlobProperties( this.parameters.AzureArchiveContainer, manifestName, - getAzureCreds(this), ); assert.strictEqual(manifestProperties.accessTier, this.parameters.AzureArchiveManifestTier); }); @@ -218,10 +198,9 @@ Then('tar access tier should be valid for object {string}', async function (this ); assert(tarName); // manifest access tier - const packProperties = await AzureHelper.getBlobProperties( + const packProperties = await this.azureClient.getBlobProperties( this.parameters.AzureArchiveContainer, tarName, - getAzureCreds(this), ); assert.strictEqual(packProperties.accessTier, this.parameters.AzureArchiveAccessTier); }); @@ -296,13 +275,11 @@ Then('blob for object {string} must be rehydrated', async function (this: Zenko, objectName: string) { const tarName = await isObjectRehydrated(this, objectName); assert(tarName); - const sent = await AzureHelper.sendBlobCreatedEventToQueue( + await this.azureClient.sendBlobCreatedEventToQueue( this.parameters.AzureArchiveQueue, this.parameters.AzureArchiveContainer, `rehydrate/${tarName}`, - getAzureCreds(this), ); - assert.strictEqual(sent, true, `Failed to send BlobCreatedEvent for ${tarName}, object ${objectName}`); }); Then('restoration of object {string} failed and ends up in DLQ', diff --git a/tests/functional/ctst/world/Zenko.ts b/tests/functional/ctst/world/Zenko.ts index 9e1667673b..80c9279142 100644 --- a/tests/functional/ctst/world/Zenko.ts +++ b/tests/functional/ctst/world/Zenko.ts @@ -23,6 +23,7 @@ import { } from 'cli-testing'; import { extractPropertyFromResults } from '../common/utils'; +import AzureClient from 'clients/azure'; import ZenkoDrctl from 'steps/dr/drctl'; import assert from 'assert'; @@ -124,6 +125,18 @@ export default class Zenko extends World { public zenkoDrCtl: ZenkoDrctl | null = null; + private _azureClient: AzureClient | null = null; + + public get azureClient(): AzureClient { + if (!this._azureClient) { + this._azureClient = new AzureClient({ + accountName: this.parameters.AzureAccountName, + accountKey: this.parameters.AzureAccountKey, + }); + } + return this._azureClient; + } + static sites: { [key: string]: { accountName: string; diff --git a/tests/functional/eslint.config.mjs b/tests/functional/eslint.config.mjs index 2f64995db5..f1e036c343 100644 --- a/tests/functional/eslint.config.mjs +++ b/tests/functional/eslint.config.mjs @@ -38,6 +38,19 @@ export default tseslint.config( }, }, + // TypeScript files (shared clients) + { + files: ['clients/**/*.ts'], + extends: [ + ...compat.extends('scality'), + ...tseslint.configs.recommended, + ], + rules: { + // CLI setup scripts report progress to the console + 'no-console': 'off', + }, + }, + // JavaScript files (mocha) { files: ['mocha/**/*.js'], diff --git a/tests/functional/package.json b/tests/functional/package.json index 3f34ac365c..336fab0033 100644 --- a/tests/functional/package.json +++ b/tests/functional/package.json @@ -10,8 +10,8 @@ "@aws-sdk/client-iam": "^3.901.0", "@aws-sdk/client-s3": "^3.931.0", "@aws-sdk/client-sts": "^3.901.0", - "@azure/storage-blob": "^12.12.0", - "@azure/storage-queue": "^12.30.0", + "@azure/storage-blob": "^12.33.0", + "@azure/storage-queue": "^12.31.0", "@cucumber/cucumber": "^13.2.0", "@eslint/compat": "^2.1.0", "@eslint/eslintrc": "^3.3.6", @@ -70,8 +70,5 @@ "test:all_extensions": "run-p --aggregate-output test:crr test:aws_crr test:expiration test:transition test:ingestion_oob_s3c", "test:object_api": "MOCHA_FILE=_reports/object-api.xml mocha --config mocha/.mocharc.js -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=mocha/mocha-reporter.js --recursive mocha/cloudserver/keyFormatVersion/tests", "lint": "eslint ." - }, - "resolutions": { - "@azure/core-client": "1.10.1" } } diff --git a/tests/functional/testResourcesSetup/index.ts b/tests/functional/testResourcesSetup/index.ts index 4a30266b02..5990d02c5c 100644 --- a/tests/functional/testResourcesSetup/index.ts +++ b/tests/functional/testResourcesSetup/index.ts @@ -1,8 +1,8 @@ import { STSClient } from '@aws-sdk/client-sts'; import { CoreV1Api } from '@kubernetes/client-node'; import { loadConfig, loadEnv, Env } from './config'; -import { PensieveClient } from './setup/clients/pensieveClient'; -import { createK8sClient } from './setup/clients/k8s'; +import { PensieveClient } from 'clients/pensieveClient'; +import { createK8sClient } from 'clients/k8s'; import { createAccounts } from './setup/accounts'; import { createEndpoints } from './setup/endpoints'; import { createLocations } from './setup/locations'; diff --git a/tests/functional/testResourcesSetup/setup/accounts.ts b/tests/functional/testResourcesSetup/setup/accounts.ts index 778b5cd397..fdb6ffbb24 100644 --- a/tests/functional/testResourcesSetup/setup/accounts.ts +++ b/tests/functional/testResourcesSetup/setup/accounts.ts @@ -1,7 +1,7 @@ import { STSClient, AssumeRoleWithWebIdentityCommand } from '@aws-sdk/client-sts'; import { CoreV1Api } from '@kubernetes/client-node'; -import { PensieveClient } from './clients/pensieveClient'; -import { AccountCredentials, createKubernetesSecret } from './clients/k8s'; +import { PensieveClient } from 'clients/pensieveClient'; +import { AccountCredentials, createKubernetesSecret } from 'clients/k8s'; import { Env } from '../config'; async function getAccountCredentials( diff --git a/tests/functional/testResourcesSetup/setup/endpoints.ts b/tests/functional/testResourcesSetup/setup/endpoints.ts index e009424d04..1d61c95141 100644 --- a/tests/functional/testResourcesSetup/setup/endpoints.ts +++ b/tests/functional/testResourcesSetup/setup/endpoints.ts @@ -1,4 +1,4 @@ -import { PensieveClient } from './clients/pensieveClient'; +import { PensieveClient } from 'clients/pensieveClient'; import { EndpointConfig, Env } from '../config'; export async function createEndpoints( diff --git a/tests/functional/testResourcesSetup/setup/locations.ts b/tests/functional/testResourcesSetup/setup/locations.ts index b4b377cffe..25e08e870f 100644 --- a/tests/functional/testResourcesSetup/setup/locations.ts +++ b/tests/functional/testResourcesSetup/setup/locations.ts @@ -6,9 +6,9 @@ import { CreatePolicyCommand, AttachRolePolicyCommand, } from '@aws-sdk/client-iam'; -import { PensieveClient } from './clients/pensieveClient'; +import { PensieveClient } from 'clients/pensieveClient'; import { LocationConfig, Env } from '../config'; -import { AccountCredentials } from './clients/k8s'; +import { AccountCredentials } from 'clients/k8s'; interface CRRUserCredentials { accessKey: string; diff --git a/tests/functional/tsconfig.json b/tests/functional/tsconfig.json index 0b52c51dd1..0b6f33753e 100644 --- a/tests/functional/tsconfig.json +++ b/tests/functional/tsconfig.json @@ -15,6 +15,7 @@ "baseUrl": "./", "paths": { "cli-testing": ["./node_modules/cli-testing"], + "clients/*": ["./clients/*"], "common/*": ["./ctst/common/*"], "steps/*": ["./ctst/steps/*"], "world/*": ["./ctst/world/*"] @@ -29,6 +30,7 @@ "ignore": ["(?:^|/)node_modules/(?!cli-testing)"] }, "include": [ + "clients/**/*", "ctst/**/*", "testResourcesSetup/**/*" ], diff --git a/tests/functional/yarn.lock b/tests/functional/yarn.lock index 024f8015b0..bf8e9a6f90 100644 --- a/tests/functional/yarn.lock +++ b/tests/functional/yarn.lock @@ -388,7 +388,7 @@ "@azure/core-util" "^1.13.0" tslib "^2.6.2" -"@azure/core-client@1.10.1", "@azure/core-client@^1.9.2", "@azure/core-client@^1.9.3": +"@azure/core-client@^1.9.2", "@azure/core-client@^1.9.3": version "1.10.1" resolved "https://registry.yarnpkg.com/@azure/core-client/-/core-client-1.10.1.tgz#83d78f97d647ab22e6811a7a68bb4223e7a1d019" integrity sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w== @@ -533,7 +533,7 @@ events "^3.0.0" tslib "^2.8.1" -"@azure/storage-blob@^12.12.0", "@azure/storage-blob@^12.31.0": +"@azure/storage-blob@^12.31.0", "@azure/storage-blob@^12.33.0": version "12.33.0" resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.33.0.tgz#10ad4586f4922731b8b79cc86becada7567a1fab" integrity sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA== @@ -586,7 +586,7 @@ "@azure/storage-common" "^12.0.0" tslib "^2.8.1" -"@azure/storage-queue@^12.30.0": +"@azure/storage-queue@^12.31.0": version "12.31.0" resolved "https://registry.yarnpkg.com/@azure/storage-queue/-/storage-queue-12.31.0.tgz#af3615bf59614476b953c11037cf1c7c2b20cfe5" integrity sha512-OzhvKO7G8+EXkUQPbKFkO7JNIAMkQZ9gMyZ85dsTCOk8jRRCbGbn3wsk3qrCAf6XWm+E2/fUsdLbX4YCOAWEKw==