Integrate Photon in AzCopy

azure-pipelines.yml

@@ -167,7 +167,7 @@ jobs:
           # setting this before will always work since it is valid to break an un-acquired mutex.
           echo '##vso[task.setvariable variable=isMutexSet]true'
           # acquire the mutex before running live tests to avoid conflicts
-          python ./tool_distributed_mutex.py lock "$(MUTEX_URL)"
+          python ./tool_distributed_mutex.py lock "$(MUTEX1_URL)"
         name: 'Acquire_the_distributed_mutex'
       - template: azurePipelineTemplates/run-ut.yml
         parameters:
@@ -241,7 +241,7 @@ jobs:
           summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml
       - script: |
           pip install azure-storage-blob==12.12.0
-          python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)"
+          python ./tool_distributed_mutex.py unlock "$(MUTEX1_URL)"
         name: 'Release_the_distributed_mutex'
         # this runs even if the job was canceled (only if the mutex was acquired by this job)
         condition: and(always(), eq(variables['isMutexSet'], 'true'))

azurePipelineTemplates/run-e2e.yml

@@ -48,7 +48,7 @@ jobs:
       - bash: |
           npm install -g azurite
           mkdir azurite
-          azurite --silent --location azurite --debug azurite\debug.log &
+          azurite --silent --skipApiVersionCheck --location azurite --debug azurite\debug.log &
         displayName: 'Install and Run Azurite'
       # Running E2E Tests on AMD64
       - task: AzureCLI@2

cmd/zt_copy_blob_download_test.go

@@ -999,8 +999,9 @@ func TestListOfVersions(t *testing.T) {
 
 	// confirm that base blob has 2 versions
 	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Prefix:  to.Ptr(blobName),
-		Include: container.ListBlobsInclude{Versions: true},
+		Prefix:         to.Ptr(blobName),
+		Include:        container.ListBlobsInclude{Versions: true},
+		UseArrowFormat: to.Ptr(true),
 	})
 	list, err := pager.NextPage(ctx)
 	a.NoError(err)

cmd/zt_overwrite_posix_properties_test.go

@@ -90,8 +90,9 @@ func TestOverwritePosixProperties(t *testing.T) {
 	})
 
 	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Include: container.ListBlobsInclude{Metadata: true, Tags: true},
-		Prefix:  to.Ptr(filepath.Base(dirPath)),
+		Include:        container.ListBlobsInclude{Metadata: true, Tags: true},
+		Prefix:         to.Ptr(filepath.Base(dirPath)),
+		UseArrowFormat: to.Ptr(true),
 	})
 	listBlob, err := pager.NextPage(context.TODO())
 

cmd/zt_remove_blob_test.go

@@ -796,8 +796,9 @@ func TestPermDeleteSnapshotsVersionsUnderSingleBlob(t *testing.T) {
 	a.Equal(3, len(blobList))
 
 	pager := cc.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Prefix:  to.Ptr(blobName),
-		Include: container.ListBlobsInclude{Deleted: true, Snapshots: true},
+		Prefix:         to.Ptr(blobName),
+		Include:        container.ListBlobsInclude{Deleted: true, Snapshots: true},
+		UseArrowFormat: to.Ptr(true),
 	})
 	list, err := pager.NextPage(ctx)
 	a.Nil(err)

cmd/zt_sync_blob_blob_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/Azure/azure-storage-azcopy/v10/common"
@@ -206,7 +207,7 @@ func TestSyncS2SWithMismatchedDestination(t *testing.T) {
 
 		// make sure the extra blobs were deleted
 		extraFilesFound := false
-		pager := dstContainerClient.NewListBlobsFlatPager(nil)
+		pager := dstContainerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{UseArrowFormat: to.Ptr(true)})
 		for pager.More() {
 			listResponse, err := pager.NextPage(ctx)
 			a.Nil(err)

common/access.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"context"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 )
@@ -42,7 +43,7 @@ func IsSourcePublicBlob(sourceURI string, ctx context.Context) bool {
 			panic("Container client was unable to be created.")
 		}
 
-		pager := containerClient.NewListBlobsFlatPager(nil)
+		pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{UseArrowFormat: to.Ptr(true)})
 		_, err = pager.NextPage(ctx)
 		if err == nil {
 			return true

common/environment.go

@@ -418,7 +418,7 @@ func (EnvironmentVariable) CredentialType() EnvironmentVariable {
 func (EnvironmentVariable) DefaultServiceApiVersion() EnvironmentVariable {
 	return EnvironmentVariable{
 		Name:         "AZCOPY_DEFAULT_SERVICE_API_VERSION",
-		DefaultValue: "2025-07-05",
+		DefaultValue: "2026-06-06",
 		Description:  "Overrides the service API version so that AzCopy could accommodate custom environments such as Azure Stack.",
 	}
 }

common/version.go

@@ -1,6 +1,6 @@
 package common
 
-const AzcopyVersion = "10.32.2"
+const AzcopyVersion = "10.33.0~preview.1"
 const UserAgent = "AzCopy/" + AzcopyVersion
 const S3ImportUserAgent = "S3Import " + UserAgent
 const GCPImportUserAgent = "GCPImport " + UserAgent

e2etest/declarativeResourceManagers.go

@@ -23,6 +23,7 @@ package e2etest
 import (
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
@@ -334,6 +335,7 @@ func (r *resourceBlobContainer) getVersions(a asserter, objectName string) []str
 	p := r.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
 		Include: container.ListBlobsInclude{Versions: true},
 		Prefix:  &objectName,
+		UseArrowFormat: to.Ptr(true),
 	})
 
 	versions := &timestampSortable{

e2etest/newe2e_resource_managers_blob.go

@@ -263,8 +263,9 @@ func (b *BlobContainerResourceManager) ListObjects(a Asserter, prefix string, re
 
 	if !recursive {
 		pager := b.InternalClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
-			Include: container.ListBlobsInclude{Metadata: true, Tags: true, Versions: true, LegalHold: true},
-			Prefix:  &prefix,
+			Include:        container.ListBlobsInclude{Metadata: true, Tags: true, Versions: true, LegalHold: true},
+			Prefix:         &prefix,
+			UseArrowFormat: to.Ptr(true),
 		})
 
 		for pager.More() {
@@ -277,8 +278,9 @@ func (b *BlobContainerResourceManager) ListObjects(a Asserter, prefix string, re
 		}
 	} else {
 		pager := b.InternalClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-			Include: container.ListBlobsInclude{Metadata: true, Tags: true, Versions: true, LegalHold: true},
-			Prefix:  &prefix,
+			Include:        container.ListBlobsInclude{Metadata: true, Tags: true, Versions: true, LegalHold: true},
+			Prefix:         &prefix,
+			UseArrowFormat: to.Ptr(true),
 		})
 
 		for pager.More() {

e2etest/scenario_helpers.go

@@ -650,7 +650,10 @@ func (scenarioHelper) generateBlobsFromList(c asserter, options *generateBlobFro
 func (s scenarioHelper) enumerateContainerBlobProperties(a asserter, containerClient *container.Client, fileSystemURL *filesystem.Client) map[string]*objectProperties {
 	result := make(map[string]*objectProperties)
 
-	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{Include: container.ListBlobsInclude{Metadata: true, Tags: true}})
+	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Include: container.ListBlobsInclude{Metadata: true, Tags: true},
+		UseArrowFormat: to.Ptr(true),
+	})
 	for pager.More() {
 		listBlob, err := pager.NextPage(context.TODO())
 		a.AssertNoErr(err)

go.mod

@@ -1,98 +1,111 @@
 module github.com/Azure/azure-storage-azcopy/v10
 
 require (
-	cloud.google.com/go/storage v1.50.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2-0.20250507070125-9e8eb1f00372
+	cloud.google.com/go/storage v1.59.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.5-0.20260410055824-c6fa341ca22b
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4
 	github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda
-	github.com/danieljoos/wincred v1.2.2
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
+	github.com/danieljoos/wincred v1.2.3
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8
 	github.com/google/uuid v1.6.0
 	github.com/hillu/go-ntdll v0.0.0-20240418173803-69345773b582
 	github.com/mattn/go-ieproxy v0.0.12
 	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/pkg/xattr v0.4.10
+	github.com/pkg/xattr v0.4.12
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
-	github.com/spf13/cobra v1.8.1
+	github.com/spf13/cobra v1.10.2
 	github.com/wastore/keyctl v0.3.1
-	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
 	golang.org/x/oauth2 v0.34.0
-	golang.org/x/sync v0.19.0
-	golang.org/x/sys v0.40.0
-	google.golang.org/api v0.249.0
+	golang.org/x/sync v0.20.0
+	golang.org/x/sys v0.43.0
+	google.golang.org/api v0.265.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 )
 
 require github.com/stretchr/testify v1.11.1
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2
-	github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.3.0
-	github.com/Azure/go-autorest/autorest/date v0.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1
+	github.com/Azure/go-autorest/autorest/date v0.3.1
 	github.com/keybase/go-keychain v0.0.1
-	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.49.0
+	github.com/spf13/pflag v1.0.10
+	golang.org/x/net v0.53.0
 )
 
 require (
 	cel.dev/expr v0.25.1 // indirect
-	cloud.google.com/go v0.120.0 // indirect
-	cloud.google.com/go/auth v0.16.5 // indirect
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
-	cloud.google.com/go/iam v1.5.2 // indirect
-	cloud.google.com/go/monitoring v1.24.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
+	cloud.google.com/go/iam v1.5.3 // indirect
+	cloud.google.com/go/monitoring v1.24.3 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect
+	github.com/apache/arrow-go/v18 v18.5.2 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.3 // indirect
+	github.com/goccy/go-json v0.10.6 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
+	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.40.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
-	golang.org/x/time v0.13.0 // indirect
-	google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/grpc v1.79.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
+	golang.org/x/mod v0.35.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260414141209-fac6e1c83189 // indirect
+	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	golang.org/x/tools v0.44.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
+	google.golang.org/grpc v1.80.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )