From 165cea6fe994ab2a7d3ad09e82c1f7bdcf1157b5 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 25 May 2026 16:02:31 +0300 Subject: [PATCH 01/15] Add egress monitoring --- README.md | 36 ++ .../sharing/server/DeltaSharingService.scala | 97 ++++- .../sharing/server/config/ServerConfig.scala | 47 ++- .../telemetry/EgressMetricsEmitter.scala | 348 ++++++++++++++++++ .../server/DeltaSharingServiceSuite.scala | 38 ++ .../server/config/ServerConfigSuite.scala | 55 +++ .../telemetry/EgressMetricsEmitterSuite.scala | 140 +++++++ .../conf/delta-sharing-server.yaml.template | 13 + 8 files changed, 770 insertions(+), 4 deletions(-) create mode 100644 server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala create mode 100644 server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala diff --git a/README.md b/README.md index 7836ace26..6dd7dae1d 100644 --- a/README.md +++ b/README.md @@ -436,6 +436,42 @@ Download the pre-built package `delta-sharing-server-x.y.z.zip` from [GitHub Rel - Make changes to your yaml file. You may also need to update some server configs for special requirements. - To add Shared Data, add reference to Delta Lake tables you would like to share from this server in this config file. +### Optional Share Egress Metrics (Cloud Monitoring) + +The server can emit share-attributed custom metrics to Google Cloud Monitoring: + +- `custom.googleapis.com/delta_sharing/share_egress_bytes` (cumulative bytes) + +Labels emitted on this metric: + +- `share` +- `request_type` (`query` or `cdf_stream`) + +To enable this feature: + +1. Configure the `egressMetrics` block in the server yaml. +2. Ensure the runtime identity has `roles/monitoring.metricWriter` and can use + Application Default Credentials (for example `GOOGLE_APPLICATION_CREDENTIALS`). + +Example yaml: + +```yaml +shares: +- name: "share1" + schemas: + - name: "schema1" + tables: + - name: "table1" + location: "gs://my-bucket/my-table" + +egressMetrics: + enabled: true + gcpProjectId: "my-project" + cdfAggregationWindowSeconds: 60 + batchSize: 20 + flushIntervalSeconds: 15 +``` + ## Config the server to access tables on cloud storage We support sharing Delta Lake tables on S3, Azure Blob Storage and Azure Data Lake Storage Gen2. diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 873e62386..87cd51fc7 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -33,6 +33,7 @@ import com.linecorp.armeria.server.{Server, ServiceRequestContext} import com.linecorp.armeria.server.annotation.{ConsumesJson, Default, ExceptionHandler, ExceptionHandlerFunction, Get, Head, Param, Post, ProducesJson} import com.linecorp.armeria.server.auth.AuthService import io.delta.kernel.exceptions.{KernelException, TableNotFoundException} +import io.delta.standalone.internal.{DeltaResponseSingleAction => StandaloneDeltaResponseSingleAction} import io.delta.standalone.internal.DeltaCDFErrors import io.delta.standalone.internal.DeltaCDFIllegalArgumentException import io.delta.standalone.internal.DeltaDataSource @@ -44,8 +45,9 @@ import scalapb.json4s.Printer import io.delta.sharing.server.common.JsonUtils import io.delta.sharing.server.config.ServerConfig -import io.delta.sharing.server.model.{QueryStatus, SingleAction} +import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ +import io.delta.sharing.server.telemetry.{EgressMetricPoint, EgressMetricsEmitter} object ErrorCode { val UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -189,6 +191,63 @@ class DeltaSharingService(serverConfig: ServerConfig) { private val deltaSharedTableLoader = new DeltaSharedTableLoader(serverConfig) private val logger = LoggerFactory.getLogger(classOf[DeltaSharingService]) + private val egressMetricsEmitter = EgressMetricsEmitter.create(serverConfig) + private val cdfAggregationWindowMs = Option(serverConfig.getEgressMetrics) + .map(_.cdfAggregationWindowSeconds.toLong * 1000L) + .getOrElse(60000L) + private val cdfAggregates = scala.collection.mutable.HashMap.empty[(String, Long), Long] + + private def emitQueryEgressMetric(share: String, actions: Seq[Object]): Unit = { + val bytes = DeltaSharingService.extractEgressBytes(actions) + if (bytes <= 0) { + return + } + + val nowMs = System.currentTimeMillis() + val point = EgressMetricPoint( + share = share, + requestType = EgressMetricsEmitter.QueryRequestType, + egressBytes = bytes, + startTimeMs = nowMs, + endTimeMs = nowMs + ) + egressMetricsEmitter.record(point) + } + + private def emitCdfEgressMetric(share: String, actions: Seq[Object]): Unit = { + val bytes = DeltaSharingService.extractEgressBytes(actions) + if (bytes <= 0) { + return + } + + val nowMs = System.currentTimeMillis() + val windowStart = (nowMs / cdfAggregationWindowMs) * cdfAggregationWindowMs + + cdfAggregates.synchronized { + flushExpiredCdfAggregates(nowMs) + val key = (share, windowStart) + val current = cdfAggregates.getOrElse(key, 0L) + cdfAggregates.update(key, current + bytes) + } + } + + private def flushExpiredCdfAggregates(nowMs: Long): Unit = { + val expiredKeys = cdfAggregates.keys.filter { case (_, startMs) => + startMs + cdfAggregationWindowMs <= nowMs + }.toSeq + expiredKeys.foreach { key => + val (share, startMs) = key + val bytes = cdfAggregates.remove(key).get + val point = EgressMetricPoint( + share = share, + requestType = EgressMetricsEmitter.CdfStreamRequestType, + egressBytes = bytes, + startTimeMs = startMs, + endTimeMs = startMs + cdfAggregationWindowMs + ) + egressMetricsEmitter.record(point) + } + } private def logCdfRequestComplete( wallStartNs: Long, @@ -475,6 +534,8 @@ class DeltaSharingService(serverConfig: ServerConfig) { ) } + emitQueryEgressMetric(share, queryResult.actions) + streamingOutput(Some(queryResult.version), queryResult.responseFormat, queryResult.actions) } } @@ -625,6 +686,8 @@ class DeltaSharingService(serverConfig: ServerConfig) { s"You can only query table data since version ${tableConfig.startVersion}." ) } + + emitQueryEgressMetric(share, queryResult.actions) logTableQueryComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), @@ -683,6 +746,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { deltaLogUpdateNs = updateNs, requestTimeoutSecondsForLogging = Some(serverConfig.requestTimeoutSeconds) ) + emitCdfEgressMetric(share, queryResult.actions) logCdfRequestComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), @@ -739,6 +803,37 @@ object DeltaSharingService { val DELTA_SHARING_READER_FEATURES = "readerfeatures" val DELTA_SHARING_CAPABILITIES_DELIMITER = ";" + private[server] def extractEgressBytes(actions: Seq[Object]): Long = { + actions.map(extractActionBytes).sum + } + + private def extractActionBytes(action: Object): Long = { + action match { + case singleAction: SingleAction => + val raw = singleAction.unwrap + raw match { + case addFile: AddFile => addFile.size + case addFileForCDF: AddFileForCDF => addFileForCDF.size + case addCDCFile: AddCDCFile => addCDCFile.size + case _ => 0L + } + case deltaResponse: StandaloneDeltaResponseSingleAction => + Option(deltaResponse.file) + .flatMap(f => Option(f.deltaSingleAction)) + .map { deltaAction => + if (deltaAction.add != null) { + deltaAction.add.size + } else if (deltaAction.cdc != null) { + deltaAction.cdc.size + } else { + 0L + } + } + .getOrElse(0L) + case _ => 0L + } + } + private val parser = { val parser = ArgumentParsers .newFor("Delta Sharing Server") diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index 76eb63526..ccdddaa39 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -67,7 +67,9 @@ case class ServerConfig( // The TTL of the refresh token generated in queryTable API (in milliseconds). @BeanProperty var refreshTokenTtlMs: Int, // Whether to emit performance/timing log lines for table queries and CDF requests. - @BeanProperty var perfLoggingEnabled: Boolean + @BeanProperty var perfLoggingEnabled: Boolean, + // Optional telemetry settings for share-attributed egress metrics. + @BeanProperty var egressMetrics: EgressMetricsConfig ) extends ConfigItem { import ServerConfig._ @@ -91,7 +93,8 @@ case class ServerConfig( queryTablePageSizeLimit = 10000, queryTablePageTokenTtlMs = 259200000, // 3 days refreshTokenTtlMs = 3600000, // 1 hour - perfLoggingEnabled = true + perfLoggingEnabled = true, + egressMetrics = null ) } @@ -122,6 +125,44 @@ case class ServerConfig( if (ssl != null) { ssl.checkConfig() } + if (egressMetrics != null) { + egressMetrics.checkConfig() + } + } +} + +case class EgressMetricsConfig( + @BeanProperty var enabled: Boolean, + @BeanProperty var gcpProjectId: String, + @BeanProperty var cdfAggregationWindowSeconds: Int, + @BeanProperty var batchSize: Int, + @BeanProperty var flushIntervalSeconds: Int) extends ConfigItem { + + def this() = { + this( + enabled = false, + gcpProjectId = null, + cdfAggregationWindowSeconds = 60, + batchSize = 20, + flushIntervalSeconds = 15 + ) + } + + override def checkConfig(): Unit = { + if (enabled && (gcpProjectId == null || gcpProjectId.trim.isEmpty)) { + throw new IllegalArgumentException("'gcpProjectId' in 'egressMetrics' must be provided") + } + if (cdfAggregationWindowSeconds <= 0) { + throw new IllegalArgumentException( + "'cdfAggregationWindowSeconds' in 'egressMetrics' must be greater than 0") + } + if (batchSize <= 0) { + throw new IllegalArgumentException("'batchSize' in 'egressMetrics' must be greater than 0") + } + if (flushIntervalSeconds <= 0) { + throw new IllegalArgumentException( + "'flushIntervalSeconds' in 'egressMetrics' must be greater than 0") + } } } @@ -238,7 +279,7 @@ case class TableConfig( @BeanProperty var location: String, @BeanProperty var id: String = "", @BeanProperty var historyShared: Boolean = false, - @BeanProperty var startVersion: Long = 0) extends ConfigItem { + @BeanProperty var startVersion: Long = 0) extends ConfigItem { def this() { this(null, null, null) diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala new file mode 100644 index 000000000..81a2293c2 --- /dev/null +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala @@ -0,0 +1,348 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import java.io.OutputStreamWriter +import java.net.{HttpURLConnection, URL} +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.Collections +import java.util.concurrent.{Executors, ScheduledExecutorService, ThreadFactory, TimeUnit} +import java.util.concurrent.atomic.AtomicBoolean + +import scala.collection.mutable.ArrayBuffer +import scala.util.Random + +import com.google.auth.oauth2.GoogleCredentials +import org.slf4j.LoggerFactory + +import io.delta.sharing.server.common.JsonUtils +import io.delta.sharing.server.config.{EgressMetricsConfig, ServerConfig} + +case class EgressMetricPoint( + share: String, + requestType: String, + egressBytes: Long, + startTimeMs: Long, + endTimeMs: Long) + +trait EgressMetricsEmitter { + def record(point: EgressMetricPoint): Unit +} + +trait MonitoringTimeSeriesClient { + def write(timeSeries: Seq[Map[String, Any]]): Int +} + +private class HttpMonitoringTimeSeriesClient(config: EgressMetricsConfig) + extends MonitoringTimeSeriesClient { + + private val scope = Collections.singletonList("https://www.googleapis.com/auth/monitoring.write") + private val metricWriteUrl = + s"https://monitoring.googleapis.com/v3/projects/${config.gcpProjectId}/timeSeries" + private var credentials: GoogleCredentials = _ + + override def write(timeSeries: Seq[Map[String, Any]]): Int = { + val body = JsonUtils.toJson(Map("timeSeries" -> timeSeries)) + val connection = new URL(metricWriteUrl).openConnection().asInstanceOf[HttpURLConnection] + try { + connection.setRequestMethod("POST") + connection.setConnectTimeout(3000) + connection.setReadTimeout(5000) + connection.setDoOutput(true) + connection.setRequestProperty("Authorization", s"Bearer ${accessTokenValue}") + connection.setRequestProperty("Content-Type", "application/json; charset=utf-8") + + val writer = new OutputStreamWriter(connection.getOutputStream, UTF_8) + try { + writer.write(body) + } finally { + writer.close() + } + connection.getResponseCode + } finally { + connection.disconnect() + } + } + + private def accessTokenValue: String = synchronized { + if (credentials == null) { + credentials = GoogleCredentials.getApplicationDefault().createScoped(scope) + } + val token = credentials.getAccessToken + if (token == null || + token.getExpirationTime == null || + token.getExpirationTime.getTime <= (System.currentTimeMillis() + 60000L)) { + credentials.refresh() + } + credentials.getAccessToken.getTokenValue + } +} + +object EgressMetricsEmitter { + val ShareEgressBytesMetric = "custom.googleapis.com/delta_sharing/share_egress_bytes" + val UnknownShare = "unknown" + val QueryRequestType = "query" + val CdfStreamRequestType = "cdf_stream" + + def create(serverConfig: ServerConfig): EgressMetricsEmitter = { + val cfg = Option(serverConfig.getEgressMetrics) + cfg match { + case Some(c) if c.enabled => new GcpCloudMonitoringEgressMetricsEmitter(c) + case _ => NoopEgressMetricsEmitter + } + } +} + +object NoopEgressMetricsEmitter extends EgressMetricsEmitter { + override def record(point: EgressMetricPoint): Unit = {} +} + +private case class CumulativeTotals(var bytes: Long, var seriesStartMs: Long) + +private sealed trait SendResult +private case object SendSuccess extends SendResult +private case object SendRetryableFailure extends SendResult +private case class SendNonRetryableFailure(status: Int) extends SendResult + +class GcpCloudMonitoringEgressMetricsEmitter( + config: EgressMetricsConfig, + client: MonitoringTimeSeriesClient, + scheduler: ScheduledExecutorService, + registerShutdownHook: Boolean, + maxPendingSeries: Int, + maxRetryAttempts: Int, + retryBaseDelayMs: Long) extends EgressMetricsEmitter { + + def this(config: EgressMetricsConfig) = { + this( + config = config, + client = new HttpMonitoringTimeSeriesClient(config), + scheduler = GcpCloudMonitoringEgressMetricsEmitter.newDefaultScheduler(), + registerShutdownHook = true, + maxPendingSeries = math.max(config.batchSize * 200, config.batchSize), + maxRetryAttempts = 3, + retryBaseDelayMs = 200L + ) + } + + import EgressMetricsEmitter._ + + private val logger = LoggerFactory.getLogger(classOf[GcpCloudMonitoringEgressMetricsEmitter]) + private val cumulativeByLabel = + scala.collection.mutable.HashMap.empty[String, CumulativeTotals] + private val pendingSeries = ArrayBuffer.empty[Map[String, Any]] + private val flushInFlight = new AtomicBoolean(false) + private val random = new Random() + private var droppedSeriesCount = 0L + + scheduler.scheduleAtFixedRate( + new Runnable { + override def run(): Unit = { + triggerAsyncFlush() + } + }, + config.flushIntervalSeconds, + config.flushIntervalSeconds, + TimeUnit.SECONDS) + + if (registerShutdownHook) { + // scalastyle:off runtimeaddshutdownhook + Runtime.getRuntime.addShutdownHook(new Thread(new Runnable { + override def run(): Unit = { + try { + flushPendingSynchronously() + } finally { + scheduler.shutdownNow() + } + } + })) + // scalastyle:on runtimeaddshutdownhook + } + + override def record(point: EgressMetricPoint): Unit = synchronized { + val share = sanitizeLabel(point.share, UnknownShare) + val requestType = sanitizeLabel(point.requestType, QueryRequestType) + val key = s"$share|$requestType" + val startMs = if (point.startTimeMs > 0) point.startTimeMs else System.currentTimeMillis() + val totals = cumulativeByLabel.getOrElseUpdate(key, CumulativeTotals(0L, startMs)) + totals.bytes += point.egressBytes + val endMs = math.max(point.endTimeMs, System.currentTimeMillis()) + + pendingSeries += makeTimeSeries( + ShareEgressBytesMetric, + share, + requestType, + totals.seriesStartMs, + endMs, + Map("int64Value" -> totals.bytes.toString)) + + if (pendingSeries.size > maxPendingSeries) { + val overflow = pendingSeries.size - maxPendingSeries + pendingSeries.remove(0, overflow) + droppedSeriesCount += overflow + if (droppedSeriesCount % 100 == 0) { + logger.warn( + s"Dropped $droppedSeriesCount queued egress metric points due to queue overload") + } + } + + if (pendingSeries.size >= config.batchSize) { + triggerAsyncFlush() + } + } + + private def triggerAsyncFlush(): Unit = { + if (flushInFlight.compareAndSet(false, true)) { + scheduler.execute(new Runnable { + override def run(): Unit = { + try { + flushPendingSynchronously() + } finally { + flushInFlight.set(false) + } + } + }) + } + } + + private[telemetry] def snapshotPendingSeriesSize(): Int = synchronized { + pendingSeries.size + } + + private def sanitizeLabel(value: String, fallback: String): String = { + if (value == null || value.trim.isEmpty) fallback else value.trim + } + + private def toRfc3339(ms: Long): String = { + java.time.Instant.ofEpochMilli(ms).toString + } + + private def makeTimeSeries( + metricType: String, + share: String, + requestType: String, + startTimeMs: Long, + endTimeMs: Long, + value: Map[String, Any]): Map[String, Any] = { + Map( + "metric" -> Map( + "type" -> metricType, + "labels" -> Map( + "share" -> share, + "request_type" -> requestType + ) + ), + "resource" -> Map( + "type" -> "global", + "labels" -> Map("project_id" -> config.gcpProjectId) + ), + "points" -> Seq(Map( + "interval" -> Map( + "startTime" -> toRfc3339(startTimeMs), + "endTime" -> toRfc3339(endTimeMs) + ), + "value" -> value + )) + ) + } + + private def flushPendingSynchronously(): Unit = { + val snapshot = synchronized { + if (pendingSeries.isEmpty) { + Seq.empty[Map[String, Any]] + } else { + val batch = pendingSeries.toVector + pendingSeries.clear() + batch + } + } + if (snapshot.isEmpty) { + return + } + + val batches = snapshot.grouped(config.batchSize).toSeq + batches.foreach { batch => + handleBatchWrite(batch) + } + } + + private def handleBatchWrite(batch: Seq[Map[String, Any]]): Unit = { + sendWithRetries(batch) match { + case SendSuccess => + case SendRetryableFailure => + synchronized { + pendingSeries.prependAll(batch) + } + case SendNonRetryableFailure(status) if batch.size > 1 => + val (left, right) = batch.splitAt(batch.size / 2) + handleBatchWrite(left) + handleBatchWrite(right) + case SendNonRetryableFailure(status) => + logger.warn(s"Dropping non-retryable egress metric point. status=$status") + } + } + + private def sendWithRetries(batch: Seq[Map[String, Any]]): SendResult = { + var attempt = 1 + while (attempt <= maxRetryAttempts) { + try { + val status = client.write(batch) + if (status / 100 == 2) { + return SendSuccess + } + if (!isRetryableStatus(status)) { + return SendNonRetryableFailure(status) + } + if (attempt < maxRetryAttempts) { + backoffSleep(attempt) + } + } catch { + case e: Throwable => + logger.warn( + s"Failed to write egress metrics (attempt=$attempt/$maxRetryAttempts)", + e) + if (attempt < maxRetryAttempts) { + backoffSleep(attempt) + } + } + attempt += 1 + } + SendRetryableFailure + } + + private def isRetryableStatus(status: Int): Boolean = { + status == 429 || status == 500 || status == 502 || status == 503 || status == 504 + } + + private def backoffSleep(attempt: Int): Unit = { + val jitter = random.nextInt(100) + val delay = retryBaseDelayMs * (1L << (attempt - 1)) + jitter + Thread.sleep(delay) + } +} + +object GcpCloudMonitoringEgressMetricsEmitter { + private[telemetry] def newDefaultScheduler(): ScheduledExecutorService = { + Executors.newSingleThreadScheduledExecutor(new ThreadFactory { + override def newThread(r: Runnable): Thread = { + val t = new Thread(r, "delta-sharing-egress-metrics") + t.setDaemon(true) + t + } + }) + } +} diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index e034a62b0..5eb8aaabf 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -232,6 +232,44 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { }.getMessage.contains("endingVersion is not a valid number") } + test("extractEgressBytes") { + val actions: Seq[Object] = Seq( + AddFile( + url = "u1", + id = "1", + partitionValues = Map.empty, + size = 10L + ).wrap, + AddFileForCDF( + url = "u2", + id = "2", + partitionValues = Map.empty, + size = 15L, + version = 1L, + timestamp = 1L + ).wrap, + AddCDCFile( + url = "u3", + id = "3", + partitionValues = Map.empty, + size = 20L, + version = 1L, + timestamp = 1L + ).wrap, + RemoveFile( + url = "u4", + id = "4", + partitionValues = Map.empty, + size = 25L, + version = 1L, + timestamp = 1L + ).wrap, + QueryStatus(queryId = "q1").wrap + ) + + assert(DeltaSharingService.extractEgressBytes(actions) == 45L) + } + integrationTest("401 Unauthorized Error: incorrect token") { val url = requestPath("/shares") val connection = new URL(url).openConnection().asInstanceOf[HttpsURLConnection] diff --git a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala index 5eae5cacb..7e75385b8 100644 --- a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala @@ -98,12 +98,59 @@ class ServerConfigSuite extends FunSuite { serverConfig.setVersion(1) serverConfig.setShares(sharesInTemplate) serverConfig.setPort(8080) + val egressMetrics = new EgressMetricsConfig() + egressMetrics.setEnabled(false) + egressMetrics.setGcpProjectId("") + egressMetrics.setCdfAggregationWindowSeconds(60) + egressMetrics.setBatchSize(20) + egressMetrics.setFlushIntervalSeconds(15) + serverConfig.setEgressMetrics(egressMetrics) assert(loaded == serverConfig) } finally { tempFile.delete() } } + test("egress metrics config") { + val serverConfig = new ServerConfig() + serverConfig.setVersion(1) + serverConfig.setShares(Arrays.asList( + ShareConfig("share1", Arrays.asList( + SchemaConfig("schema1", Arrays.asList( + TableConfig( + name = "table1", + location = "s3a://bucket/path", + id = null + ) + )) + )) + )) + val egressMetrics = new EgressMetricsConfig() + egressMetrics.setEnabled(true) + egressMetrics.setGcpProjectId("project-1") + egressMetrics.setCdfAggregationWindowSeconds(60) + egressMetrics.setBatchSize(10) + egressMetrics.setFlushIntervalSeconds(5) + serverConfig.setEgressMetrics(egressMetrics) + testConfig( + """version: 1 + |shares: + |- name: share1 + | schemas: + | - name: schema1 + | tables: + | - name: table1 + | location: s3a://bucket/path + |egressMetrics: + | enabled: true + | gcpProjectId: project-1 + | cdfAggregationWindowSeconds: 60 + | batchSize: 10 + | flushIntervalSeconds: 5 + |""".stripMargin, + serverConfig) + } + test("accept unknown fields") { val serverConfig = new ServerConfig() serverConfig.setVersion(1) @@ -158,6 +205,14 @@ class ServerConfigSuite extends FunSuite { } } + test("EgressMetricsConfig") { + assertInvalidConfig("'gcpProjectId' in 'egressMetrics' must be provided") { + val e = new EgressMetricsConfig() + e.setEnabled(true) + e.checkConfig() + } + } + test("SSLConfig") { assertInvalidConfig("'certificateFile' in a SSL config must be provided") { val s = new SSLConfig() diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala new file mode 100644 index 000000000..9dad4b068 --- /dev/null +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala @@ -0,0 +1,140 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import java.util.concurrent.{Executors, ScheduledExecutorService} + +import org.scalatest.FunSuite + +import io.delta.sharing.server.config.EgressMetricsConfig + +class EgressMetricsEmitterSuite extends FunSuite { + + private class FakeClient(sleepMs: Long = 0L) extends MonitoringTimeSeriesClient { + val writes = scala.collection.mutable.ArrayBuffer.empty[Seq[Map[String, Any]]] + + override def write(timeSeries: Seq[Map[String, Any]]): Int = { + if (sleepMs > 0) { + Thread.sleep(sleepMs) + } + writes.synchronized { + writes += timeSeries + } + 200 + } + + def writeCount: Int = writes.synchronized { + writes.size + } + } + + private def mkConfig(batchSize: Int, flushIntervalSeconds: Int): EgressMetricsConfig = { + val cfg = new EgressMetricsConfig() + cfg.setEnabled(true) + cfg.setGcpProjectId("test-project") + cfg.setBatchSize(batchSize) + cfg.setFlushIntervalSeconds(flushIntervalSeconds) + cfg.setCdfAggregationWindowSeconds(60) + cfg + } + + private def waitUntil(condition: => Boolean, timeoutMs: Long): Unit = { + val deadline = System.currentTimeMillis() + timeoutMs + while (!condition && System.currentTimeMillis() < deadline) { + Thread.sleep(20) + } + assert(condition) + } + + private def newEmitter( + config: EgressMetricsConfig, + client: MonitoringTimeSeriesClient, + scheduler: ScheduledExecutorService): GcpCloudMonitoringEgressMetricsEmitter = { + new GcpCloudMonitoringEgressMetricsEmitter( + config = config, + client = client, + scheduler = scheduler, + registerShutdownHook = false, + maxPendingSeries = 1000, + maxRetryAttempts = 2, + retryBaseDelayMs = 10L) + } + + private def extractStartTimes(client: FakeClient): Seq[String] = { + client.writes.synchronized { + client.writes.flatMap { batch => + batch.map { timeSeries => + val points = timeSeries("points").asInstanceOf[Seq[Map[String, Any]]] + val interval = points.head("interval").asInstanceOf[Map[String, Any]] + interval("startTime").asInstanceOf[String] + } + }.toSeq + } + } + + test("cumulative metric keeps stable start time for a series") { + val scheduler = Executors.newSingleThreadScheduledExecutor() + try { + val client = new FakeClient() + val emitter = newEmitter(mkConfig(batchSize = 1, flushIntervalSeconds = 60), client, scheduler) + + emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) + emitter.record(EgressMetricPoint("share-a", "query", 20L, 7000L, 8000L)) + + waitUntil(client.writeCount >= 2, timeoutMs = 3000) + val starts = extractStartTimes(client) + assert(starts.nonEmpty) + assert(starts.forall(_ == starts.head)) + } finally { + scheduler.shutdownNow() + } + } + + test("low traffic still flushes via scheduled flush") { + val scheduler = Executors.newSingleThreadScheduledExecutor() + try { + val client = new FakeClient() + val emitter = newEmitter(mkConfig(batchSize = 10, flushIntervalSeconds = 1), client, scheduler) + + emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) + + waitUntil(client.writeCount >= 1, timeoutMs = 4000) + } finally { + scheduler.shutdownNow() + } + } + + test("record path is non-blocking even when monitoring write is slow") { + val scheduler = Executors.newSingleThreadScheduledExecutor() + try { + val slowClient = new FakeClient(sleepMs = 500) + val emitter = newEmitter( + mkConfig(batchSize = 1, flushIntervalSeconds = 60), + slowClient, + scheduler) + + val startNs = System.nanoTime() + emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) + val elapsedMs = (System.nanoTime() - startNs) / 1000000L + + assert(elapsedMs < 250L) + waitUntil(slowClient.writeCount >= 1, timeoutMs = 4000) + } finally { + scheduler.shutdownNow() + } + } +} diff --git a/server/src/universal/conf/delta-sharing-server.yaml.template b/server/src/universal/conf/delta-sharing-server.yaml.template index 4fd95a9b4..e30c0dd51 100644 --- a/server/src/universal/conf/delta-sharing-server.yaml.template +++ b/server/src/universal/conf/delta-sharing-server.yaml.template @@ -65,3 +65,16 @@ queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 # The TTL of the refresh token generated in queryTable API (in milliseconds). refreshTokenTtlMs: 3600000 +# Whether to emit performance/timing log lines for table queries and CDF requests. +perfLoggingEnabled: true +# Optional share-attributed egress metrics emitted to Cloud Monitoring. +egressMetrics: + enabled: false + # Required when enabled=true. + gcpProjectId: "" + # CDF egress bytes are aggregated into this window before emission. + cdfAggregationWindowSeconds: 60 + # Max number of time series rows queued before flushing. + batchSize: 20 + # Flush cadence for queued time series writes. + flushIntervalSeconds: 15 From dc7387ca77873fe8cc8b57cd54f49336f452b7a9 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 25 May 2026 16:10:24 +0300 Subject: [PATCH 02/15] Enable egress monitoring --- manifests/base/configmap.yaml | 6 ++++++ manifests/base/deployment.yaml | 9 +++++++-- manifests/zcloud-prod/configmap.yaml | 6 ++++++ manifests/zcloud-prod2/configmap.yaml | 6 ++++++ manifests/zing-preview/configmap.yaml | 6 ++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/manifests/base/configmap.yaml b/manifests/base/configmap.yaml index 4f86d4b0a..428fd0e99 100644 --- a/manifests/base/configmap.yaml +++ b/manifests/base/configmap.yaml @@ -26,3 +26,9 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 + egressMetrics: + enabled: true + gcpProjectId: "GCP_PROJECT_ID_PLACEHOLDER" + cdfAggregationWindowSeconds: 60 + batchSize: 20 + flushIntervalSeconds: 15 diff --git a/manifests/base/deployment.yaml b/manifests/base/deployment.yaml index ef0e83750..591770f01 100644 --- a/manifests/base/deployment.yaml +++ b/manifests/base/deployment.yaml @@ -27,8 +27,8 @@ spec: - -c - | echo "Merging base config and shares config..." - # Replace bearer token placeholder in base config - sed "s/BEARER_TOKEN_PLACEHOLDER/${BEARER_TOKEN}/g" /base-config/base-config.yaml > /tmp/base.yaml + # Replace placeholders in base config + sed -e "s/BEARER_TOKEN_PLACEHOLDER/${BEARER_TOKEN}/g" -e "s/GCP_PROJECT_ID_PLACEHOLDER/${GCP_PROJECT_ID}/g" /base-config/base-config.yaml > /tmp/base.yaml # Merge base config with shares config yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' /tmp/base.yaml /shares-config/shares-config.yaml > /shared-config/delta-sharing-server-config.yaml echo "Config merged successfully" @@ -39,6 +39,11 @@ spec: secretKeyRef: name: delta-sharing-auth key: bearer-token + - name: GCP_PROJECT_ID + valueFrom: + configMapKeyRef: + name: project-common + key: PROJECT_ID volumeMounts: - name: base-config mountPath: /base-config diff --git a/manifests/zcloud-prod/configmap.yaml b/manifests/zcloud-prod/configmap.yaml index b67a515fe..9653f99c6 100644 --- a/manifests/zcloud-prod/configmap.yaml +++ b/manifests/zcloud-prod/configmap.yaml @@ -21,3 +21,9 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 + egressMetrics: + enabled: true + gcpProjectId: "zcloud-prod" + cdfAggregationWindowSeconds: 60 + batchSize: 20 + flushIntervalSeconds: 15 diff --git a/manifests/zcloud-prod2/configmap.yaml b/manifests/zcloud-prod2/configmap.yaml index b67a515fe..4d02e63b1 100644 --- a/manifests/zcloud-prod2/configmap.yaml +++ b/manifests/zcloud-prod2/configmap.yaml @@ -21,3 +21,9 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 + egressMetrics: + enabled: true + gcpProjectId: "zcloud-prod2" + cdfAggregationWindowSeconds: 60 + batchSize: 20 + flushIntervalSeconds: 15 diff --git a/manifests/zing-preview/configmap.yaml b/manifests/zing-preview/configmap.yaml index b67a515fe..7bb9a1ed6 100644 --- a/manifests/zing-preview/configmap.yaml +++ b/manifests/zing-preview/configmap.yaml @@ -21,3 +21,9 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 + egressMetrics: + enabled: true + gcpProjectId: "zing-preview" + cdfAggregationWindowSeconds: 60 + batchSize: 20 + flushIntervalSeconds: 15 From 4d46060c90770e2cb491ecd07316fe2508cfc1e7 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Tue, 26 May 2026 16:19:01 +0300 Subject: [PATCH 03/15] Log metric push error --- .../telemetry/EgressMetricsEmitter.scala | 82 +++++++++++++++---- .../telemetry/EgressMetricsEmitterSuite.scala | 19 +++-- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala index 81a2293c2..289ea7476 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala @@ -16,7 +16,7 @@ package io.delta.sharing.server.telemetry -import java.io.OutputStreamWriter +import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStreamWriter} import java.net.{HttpURLConnection, URL} import java.nio.charset.StandardCharsets.UTF_8 import java.util.Collections @@ -43,8 +43,10 @@ trait EgressMetricsEmitter { def record(point: EgressMetricPoint): Unit } +case class MonitoringWriteResult(status: Int, responseBody: Option[String] = None) + trait MonitoringTimeSeriesClient { - def write(timeSeries: Seq[Map[String, Any]]): Int + def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult } private class HttpMonitoringTimeSeriesClient(config: EgressMetricsConfig) @@ -55,7 +57,7 @@ private class HttpMonitoringTimeSeriesClient(config: EgressMetricsConfig) s"https://monitoring.googleapis.com/v3/projects/${config.gcpProjectId}/timeSeries" private var credentials: GoogleCredentials = _ - override def write(timeSeries: Seq[Map[String, Any]]): Int = { + override def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult = { val body = JsonUtils.toJson(Map("timeSeries" -> timeSeries)) val connection = new URL(metricWriteUrl).openConnection().asInstanceOf[HttpURLConnection] try { @@ -72,12 +74,41 @@ private class HttpMonitoringTimeSeriesClient(config: EgressMetricsConfig) } finally { writer.close() } - connection.getResponseCode + val status = connection.getResponseCode + MonitoringWriteResult(status, readResponseBody(connection, status)) } finally { connection.disconnect() } } + private def readResponseBody(connection: HttpURLConnection, status: Int): Option[String] = { + val stream: InputStream = + if (status / 100 == 2) { + null + } else { + val err = connection.getErrorStream + if (err != null) err else connection.getInputStream + } + if (stream == null) { + None + } else { + val reader = new BufferedReader(new InputStreamReader(stream, UTF_8)) + try { + val sb = new StringBuilder + var line = reader.readLine() + while (line != null) { + if (sb.nonEmpty) sb.append('\n') + sb.append(line) + line = reader.readLine() + } + val body = sb.toString().trim + if (body.isEmpty) None else Some(body) + } finally { + reader.close() + } + } + } + private def accessTokenValue: String = synchronized { if (credentials == null) { credentials = GoogleCredentials.getApplicationDefault().createScoped(scope) @@ -111,12 +142,13 @@ object NoopEgressMetricsEmitter extends EgressMetricsEmitter { override def record(point: EgressMetricPoint): Unit = {} } -private case class CumulativeTotals(var bytes: Long, var seriesStartMs: Long) +private case class CumulativeTotals(var bytes: Long) private sealed trait SendResult private case object SendSuccess extends SendResult private case object SendRetryableFailure extends SendResult -private case class SendNonRetryableFailure(status: Int) extends SendResult +private case class SendNonRetryableFailure(status: Int, responseBody: Option[String]) + extends SendResult class GcpCloudMonitoringEgressMetricsEmitter( config: EgressMetricsConfig, @@ -177,8 +209,7 @@ class GcpCloudMonitoringEgressMetricsEmitter( val share = sanitizeLabel(point.share, UnknownShare) val requestType = sanitizeLabel(point.requestType, QueryRequestType) val key = s"$share|$requestType" - val startMs = if (point.startTimeMs > 0) point.startTimeMs else System.currentTimeMillis() - val totals = cumulativeByLabel.getOrElseUpdate(key, CumulativeTotals(0L, startMs)) + val totals = cumulativeByLabel.getOrElseUpdate(key, CumulativeTotals(0L)) totals.bytes += point.egressBytes val endMs = math.max(point.endTimeMs, System.currentTimeMillis()) @@ -186,7 +217,6 @@ class GcpCloudMonitoringEgressMetricsEmitter( ShareEgressBytesMetric, share, requestType, - totals.seriesStartMs, endMs, Map("int64Value" -> totals.bytes.toString)) @@ -235,7 +265,6 @@ class GcpCloudMonitoringEgressMetricsEmitter( metricType: String, share: String, requestType: String, - startTimeMs: Long, endTimeMs: Long, value: Map[String, Any]): Map[String, Any] = { Map( @@ -252,7 +281,8 @@ class GcpCloudMonitoringEgressMetricsEmitter( ), "points" -> Seq(Map( "interval" -> Map( - "startTime" -> toRfc3339(startTimeMs), + // Cloud Monitoring treats this custom metric as gauge, requiring point start==end. + "startTime" -> toRfc3339(endTimeMs), "endTime" -> toRfc3339(endTimeMs) ), "value" -> value @@ -287,12 +317,14 @@ class GcpCloudMonitoringEgressMetricsEmitter( synchronized { pendingSeries.prependAll(batch) } - case SendNonRetryableFailure(status) if batch.size > 1 => + case SendNonRetryableFailure(status, responseBody) if batch.size > 1 => val (left, right) = batch.splitAt(batch.size / 2) handleBatchWrite(left) handleBatchWrite(right) - case SendNonRetryableFailure(status) => - logger.warn(s"Dropping non-retryable egress metric point. status=$status") + case SendNonRetryableFailure(status, responseBody) => + logger.warn( + s"Dropping non-retryable egress metric point. status=$status" + + formatResponseBodyForLog(responseBody)) } } @@ -300,13 +332,18 @@ class GcpCloudMonitoringEgressMetricsEmitter( var attempt = 1 while (attempt <= maxRetryAttempts) { try { - val status = client.write(batch) + val writeResult = client.write(batch) + val status = writeResult.status if (status / 100 == 2) { return SendSuccess } if (!isRetryableStatus(status)) { - return SendNonRetryableFailure(status) + return SendNonRetryableFailure(status, writeResult.responseBody) } + logger.warn( + s"Retryable egress metrics write failure status=$status " + + s"(attempt=$attempt/$maxRetryAttempts)" + + formatResponseBodyForLog(writeResult.responseBody)) if (attempt < maxRetryAttempts) { backoffSleep(attempt) } @@ -333,6 +370,19 @@ class GcpCloudMonitoringEgressMetricsEmitter( val delay = retryBaseDelayMs * (1L << (attempt - 1)) + jitter Thread.sleep(delay) } + + private def formatResponseBodyForLog(responseBody: Option[String]): String = { + responseBody match { + case Some(body) if body.nonEmpty => + val maxLen = 2000 + val normalized = body.replaceAll("\\s+", " ").trim + val truncated = if (normalized.length <= maxLen) normalized else { + normalized.substring(0, maxLen) + "..." + } + s" responseBody=$truncated" + case _ => "" + } + } } object GcpCloudMonitoringEgressMetricsEmitter { diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala index 9dad4b068..4c84d2067 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala @@ -27,14 +27,14 @@ class EgressMetricsEmitterSuite extends FunSuite { private class FakeClient(sleepMs: Long = 0L) extends MonitoringTimeSeriesClient { val writes = scala.collection.mutable.ArrayBuffer.empty[Seq[Map[String, Any]]] - override def write(timeSeries: Seq[Map[String, Any]]): Int = { + override def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult = { if (sleepMs > 0) { Thread.sleep(sleepMs) } writes.synchronized { writes += timeSeries } - 200 + MonitoringWriteResult(200) } def writeCount: Int = writes.synchronized { @@ -74,19 +74,22 @@ class EgressMetricsEmitterSuite extends FunSuite { retryBaseDelayMs = 10L) } - private def extractStartTimes(client: FakeClient): Seq[String] = { + private def extractIntervals(client: FakeClient): Seq[(String, String)] = { client.writes.synchronized { client.writes.flatMap { batch => batch.map { timeSeries => val points = timeSeries("points").asInstanceOf[Seq[Map[String, Any]]] val interval = points.head("interval").asInstanceOf[Map[String, Any]] - interval("startTime").asInstanceOf[String] + ( + interval("startTime").asInstanceOf[String], + interval("endTime").asInstanceOf[String] + ) } }.toSeq } } - test("cumulative metric keeps stable start time for a series") { + test("gauge metric points use equal start and end times") { val scheduler = Executors.newSingleThreadScheduledExecutor() try { val client = new FakeClient() @@ -96,9 +99,9 @@ class EgressMetricsEmitterSuite extends FunSuite { emitter.record(EgressMetricPoint("share-a", "query", 20L, 7000L, 8000L)) waitUntil(client.writeCount >= 2, timeoutMs = 3000) - val starts = extractStartTimes(client) - assert(starts.nonEmpty) - assert(starts.forall(_ == starts.head)) + val intervals = extractIntervals(client) + assert(intervals.nonEmpty) + assert(intervals.forall { case (start, end) => start == end }) } finally { scheduler.shutdownNow() } From 88a97b19bcb248e5b47ebc23857abbe2cf217f54 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Wed, 27 May 2026 11:25:34 +0300 Subject: [PATCH 04/15] Add logs about metering --- manifests/base/configmap.yaml | 2 + .../sharing/server/DeltaSharingService.scala | 47 +++++++- .../sharing/server/config/ServerConfig.scala | 28 ++++- .../server/telemetry/AccessLogEmitter.scala | 99 ++++++++++++++++ .../telemetry/AccessLogEmitterSuite.scala | 111 ++++++++++++++++++ 5 files changed, 278 insertions(+), 9 deletions(-) create mode 100644 server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala create mode 100644 server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala diff --git a/manifests/base/configmap.yaml b/manifests/base/configmap.yaml index 428fd0e99..313bebb3a 100644 --- a/manifests/base/configmap.yaml +++ b/manifests/base/configmap.yaml @@ -26,6 +26,8 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 + accessLogging: + enabled: true egressMetrics: enabled: true gcpProjectId: "GCP_PROJECT_ID_PLACEHOLDER" diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 87cd51fc7..4c3653798 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -47,7 +47,7 @@ import io.delta.sharing.server.common.JsonUtils import io.delta.sharing.server.config.ServerConfig import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ -import io.delta.sharing.server.telemetry.{EgressMetricPoint, EgressMetricsEmitter} +import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry, EgressMetricPoint, EgressMetricsEmitter} object ErrorCode { val UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -192,18 +192,36 @@ class DeltaSharingService(serverConfig: ServerConfig) { private val logger = LoggerFactory.getLogger(classOf[DeltaSharingService]) private val egressMetricsEmitter = EgressMetricsEmitter.create(serverConfig) + private val accessLogEmitter = AccessLogEmitter.create(serverConfig) private val cdfAggregationWindowMs = Option(serverConfig.getEgressMetrics) .map(_.cdfAggregationWindowSeconds.toLong * 1000L) .getOrElse(60000L) private val cdfAggregates = scala.collection.mutable.HashMap.empty[(String, Long), Long] - private def emitQueryEgressMetric(share: String, actions: Seq[Object]): Unit = { + private def emitQueryEgressMetric( + share: String, + schema: String, + table: String, + actions: Seq[Object]): Unit = { val bytes = DeltaSharingService.extractEgressBytes(actions) if (bytes <= 0) { return } val nowMs = System.currentTimeMillis() + + // Emit to access log (new approach) + val entry = AccessLogEntry( + share = share, + schema = schema, + table = table, + egressBytes = bytes, + requestType = AccessLogEmitter.QueryRequestType, + timestampMs = nowMs + ) + accessLogEmitter.record(entry) + + // Also emit to legacy metrics (deprecated, for backward compatibility during transition) val point = EgressMetricPoint( share = share, requestType = EgressMetricsEmitter.QueryRequestType, @@ -214,13 +232,30 @@ class DeltaSharingService(serverConfig: ServerConfig) { egressMetricsEmitter.record(point) } - private def emitCdfEgressMetric(share: String, actions: Seq[Object]): Unit = { + private def emitCdfEgressMetric( + share: String, + schema: String, + table: String, + actions: Seq[Object]): Unit = { val bytes = DeltaSharingService.extractEgressBytes(actions) if (bytes <= 0) { return } val nowMs = System.currentTimeMillis() + + // Emit to access log immediately (new approach - no aggregation) + val entry = AccessLogEntry( + share = share, + schema = schema, + table = table, + egressBytes = bytes, + requestType = AccessLogEmitter.CdfStreamRequestType, + timestampMs = nowMs + ) + accessLogEmitter.record(entry) + + // Also emit to legacy aggregated metrics (deprecated) val windowStart = (nowMs / cdfAggregationWindowMs) * cdfAggregationWindowMs cdfAggregates.synchronized { @@ -534,7 +569,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { ) } - emitQueryEgressMetric(share, queryResult.actions) + emitQueryEgressMetric(share, schema, table, queryResult.actions) streamingOutput(Some(queryResult.version), queryResult.responseFormat, queryResult.actions) } @@ -687,7 +722,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { ) } - emitQueryEgressMetric(share, queryResult.actions) + emitQueryEgressMetric(share, schema, table, queryResult.actions) logTableQueryComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), @@ -746,7 +781,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { deltaLogUpdateNs = updateNs, requestTimeoutSecondsForLogging = Some(serverConfig.requestTimeoutSeconds) ) - emitCdfEgressMetric(share, queryResult.actions) + emitCdfEgressMetric(share, schema, table, queryResult.actions) logCdfRequestComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index ccdddaa39..b88e62d7a 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -68,8 +68,10 @@ case class ServerConfig( @BeanProperty var refreshTokenTtlMs: Int, // Whether to emit performance/timing log lines for table queries and CDF requests. @BeanProperty var perfLoggingEnabled: Boolean, - // Optional telemetry settings for share-attributed egress metrics. - @BeanProperty var egressMetrics: EgressMetricsConfig + // Telemetry settings for egress metrics (deprecated, use accessLogging). + @BeanProperty var egressMetrics: EgressMetricsConfig, + // Access logging configuration for tracking share data egress via structured logs. + @BeanProperty var accessLogging: AccessLoggingConfig ) extends ConfigItem { import ServerConfig._ @@ -94,7 +96,8 @@ case class ServerConfig( queryTablePageTokenTtlMs = 259200000, // 3 days refreshTokenTtlMs = 3600000, // 1 hour perfLoggingEnabled = true, - egressMetrics = null + egressMetrics = null, + accessLogging = null ) } @@ -128,6 +131,9 @@ case class ServerConfig( if (egressMetrics != null) { egressMetrics.checkConfig() } + if (accessLogging != null) { + accessLogging.checkConfig() + } } } @@ -166,6 +172,22 @@ case class EgressMetricsConfig( } } +/** + * Configuration for access logging to track share data egress. + * When enabled, structured JSON logs are emitted for each data access. + */ +case class AccessLoggingConfig( + @BeanProperty var enabled: Boolean) extends ConfigItem { + + def this() = { + this(enabled = false) + } + + override def checkConfig(): Unit = { + // No additional validation needed - enabled is a simple boolean + } +} + object ServerConfig{ /** The version that we understand */ private val CURRENT = 1 diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala new file mode 100644 index 000000000..ee6984c82 --- /dev/null +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -0,0 +1,99 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import org.slf4j.LoggerFactory + +import io.delta.sharing.server.common.JsonUtils +import io.delta.sharing.server.config.ServerConfig + +/** + * Represents a single access log entry for share data egress tracking. + * + * @param share The name of the share being accessed + * @param schema The schema containing the table + * @param table The table being accessed + * @param egressBytes The number of bytes transferred + * @param requestType The type of request: "query" or "cdf_stream" + * @param timestampMs The timestamp of the access in milliseconds since epoch + */ +case class AccessLogEntry( + share: String, + schema: String, + table: String, + egressBytes: Long, + requestType: String, + timestampMs: Long) + +/** + * Trait for emitting access logs for share data egress tracking. + * Implementations can write to different destinations (logging, external services, etc.) + */ +trait AccessLogEmitter { + def record(entry: AccessLogEntry): Unit +} + +object AccessLogEmitter { + val QueryRequestType = "query" + val CdfStreamRequestType = "cdf_stream" + + /** + * Creates an AccessLogEmitter based on server configuration. + * Returns a JsonAccessLogEmitter if access logging is enabled, otherwise a no-op emitter. + */ + def create(serverConfig: ServerConfig): AccessLogEmitter = { + val cfg = Option(serverConfig.getAccessLogging) + cfg match { + case Some(c) if c.enabled => new JsonAccessLogEmitter() + case _ => NoopAccessLogEmitter + } + } +} + +/** + * No-op implementation for when access logging is disabled. + */ +object NoopAccessLogEmitter extends AccessLogEmitter { + override def record(entry: AccessLogEntry): Unit = {} +} + +/** + * Emits access log entries as JSON-structured log lines. + * Uses a dedicated logger that can be filtered/routed separately in Cloud Logging. + */ +class JsonAccessLogEmitter extends AccessLogEmitter { + // Dedicated logger for access logs - can be filtered in Cloud Logging by logger name + private val logger = LoggerFactory.getLogger("delta.sharing.access") + + override def record(entry: AccessLogEntry): Unit = { + if (entry.egressBytes <= 0) { + return + } + + val logPayload = Map( + "logType" -> "ACCESS_LOG", + "share" -> entry.share, + "schema" -> entry.schema, + "table" -> entry.table, + "egressBytes" -> entry.egressBytes, + "requestType" -> entry.requestType, + "timestampMs" -> entry.timestampMs + ) + + logger.info(JsonUtils.toJson(logPayload)) + } +} diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala new file mode 100644 index 000000000..2563e8c97 --- /dev/null +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala @@ -0,0 +1,111 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import org.scalatest.FunSuite + +import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} + +class AccessLogEmitterSuite extends FunSuite { + + test("AccessLogEntry captures all required fields") { + val entry = AccessLogEntry( + share = "test-share", + schema = "test-schema", + table = "test-table", + egressBytes = 1024L, + requestType = AccessLogEmitter.QueryRequestType, + timestampMs = System.currentTimeMillis() + ) + + assert(entry.share == "test-share") + assert(entry.schema == "test-schema") + assert(entry.table == "test-table") + assert(entry.egressBytes == 1024L) + assert(entry.requestType == "query") + } + + test("NoopAccessLogEmitter ignores all records") { + // Should not throw + NoopAccessLogEmitter.record(AccessLogEntry( + share = "s", + schema = "sc", + table = "t", + egressBytes = 100L, + requestType = AccessLogEmitter.QueryRequestType, + timestampMs = 0L + )) + } + + test("JsonAccessLogEmitter skips zero-byte entries") { + val emitter = new JsonAccessLogEmitter() + // Should not throw, and should skip logging + emitter.record(AccessLogEntry( + share = "s", + schema = "sc", + table = "t", + egressBytes = 0L, + requestType = AccessLogEmitter.QueryRequestType, + timestampMs = 0L + )) + } + + test("JsonAccessLogEmitter logs non-zero entries") { + val emitter = new JsonAccessLogEmitter() + // Should not throw + emitter.record(AccessLogEntry( + share = "customer-share", + schema = "analytics", + table = "events", + egressBytes = 50000L, + requestType = AccessLogEmitter.CdfStreamRequestType, + timestampMs = 1716864000000L + )) + } + + test("AccessLogEmitter.create returns NoopAccessLogEmitter when disabled") { + val config = new ServerConfig() + // accessLogging is null by default + val emitter = AccessLogEmitter.create(config) + assert(emitter == NoopAccessLogEmitter) + } + + test("AccessLogEmitter.create returns NoopAccessLogEmitter when explicitly disabled") { + val config = new ServerConfig() + val accessConfig = new AccessLoggingConfig() + accessConfig.setEnabled(false) + config.setAccessLogging(accessConfig) + + val emitter = AccessLogEmitter.create(config) + assert(emitter == NoopAccessLogEmitter) + } + + test("AccessLogEmitter.create returns JsonAccessLogEmitter when enabled") { + val config = new ServerConfig() + val accessConfig = new AccessLoggingConfig() + accessConfig.setEnabled(true) + config.setAccessLogging(accessConfig) + + val emitter = AccessLogEmitter.create(config) + assert(emitter.isInstanceOf[JsonAccessLogEmitter]) + } + + test("request type constants are defined") { + assert(AccessLogEmitter.QueryRequestType == "query") + assert(AccessLogEmitter.CdfStreamRequestType == "cdf_stream") + } +} From 38426dd66896d28bb572dff4d5ccbd96703c5ff2 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Thu, 28 May 2026 16:20:37 +0300 Subject: [PATCH 05/15] Location lookup test --- README.md | 26 +- manifests/base/configmap.yaml | 11 +- manifests/zcloud-prod/configmap.yaml | 11 +- manifests/zcloud-prod2/configmap.yaml | 11 +- manifests/zing-preview/configmap.yaml | 11 +- .../sharing/server/DeltaSharingService.scala | 280 +++++++++--- .../sharing/server/config/ServerConfig.scala | 68 ++- .../server/telemetry/AccessLogEmitter.scala | 29 +- .../telemetry/EgressMetricsEmitter.scala | 398 ------------------ .../server/DeltaSharingServiceSuite.scala | 77 +++- .../server/config/ServerConfigSuite.scala | 53 +-- .../telemetry/AccessLogEmitterSuite.scala | 14 +- .../telemetry/EgressMetricsEmitterSuite.scala | 143 ------- .../conf/delta-sharing-server.yaml.template | 28 +- 14 files changed, 448 insertions(+), 712 deletions(-) delete mode 100644 server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala delete mode 100644 server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala diff --git a/README.md b/README.md index 6dd7dae1d..28f0f8c55 100644 --- a/README.md +++ b/README.md @@ -436,22 +436,20 @@ Download the pre-built package `delta-sharing-server-x.y.z.zip` from [GitHub Rel - Make changes to your yaml file. You may also need to update some server configs for special requirements. - To add Shared Data, add reference to Delta Lake tables you would like to share from this server in this config file. -### Optional Share Egress Metrics (Cloud Monitoring) +### Optional Share Egress Access Logging -The server can emit share-attributed custom metrics to Google Cloud Monitoring: +The server can emit share-attributed structured access log entries for each query and CDF request. -- `custom.googleapis.com/delta_sharing/share_egress_bytes` (cumulative bytes) - -Labels emitted on this metric: +Fields emitted in each log line: - `share` -- `request_type` (`query` or `cdf_stream`) - -To enable this feature: +- `schema` +- `table` +- `requestType` (`query` or `cdf_stream`) +- `egressBytes` +- `timestampMs` -1. Configure the `egressMetrics` block in the server yaml. -2. Ensure the runtime identity has `roles/monitoring.metricWriter` and can use - Application Default Credentials (for example `GOOGLE_APPLICATION_CREDENTIALS`). +To enable this feature, configure the `accessLogging` block in the server yaml. Example yaml: @@ -464,12 +462,8 @@ shares: - name: "table1" location: "gs://my-bucket/my-table" -egressMetrics: +accessLogging: enabled: true - gcpProjectId: "my-project" - cdfAggregationWindowSeconds: 60 - batchSize: 20 - flushIntervalSeconds: 15 ``` ## Config the server to access tables on cloud storage diff --git a/manifests/base/configmap.yaml b/manifests/base/configmap.yaml index 313bebb3a..b6055ad20 100644 --- a/manifests/base/configmap.yaml +++ b/manifests/base/configmap.yaml @@ -28,9 +28,8 @@ data: refreshTokenTtlMs: 3600000 accessLogging: enabled: true - egressMetrics: - enabled: true - gcpProjectId: "GCP_PROJECT_ID_PLACEHOLDER" - cdfAggregationWindowSeconds: 60 - batchSize: 20 - flushIntervalSeconds: 15 + clientRegionHeader: "x-client-region" + clientRegionSubdivisionHeader: "x-client-region-subdivision" + clientIpHeader: "x-forwarded-for" + pricingGroups: {} + defaultPricingGroup: "unknown" diff --git a/manifests/zcloud-prod/configmap.yaml b/manifests/zcloud-prod/configmap.yaml index 9653f99c6..983ce87d0 100644 --- a/manifests/zcloud-prod/configmap.yaml +++ b/manifests/zcloud-prod/configmap.yaml @@ -21,9 +21,10 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 - egressMetrics: + accessLogging: enabled: true - gcpProjectId: "zcloud-prod" - cdfAggregationWindowSeconds: 60 - batchSize: 20 - flushIntervalSeconds: 15 + clientRegionHeader: "x-client-region" + clientRegionSubdivisionHeader: "x-client-region-subdivision" + clientIpHeader: "x-forwarded-for" + pricingGroups: {} + defaultPricingGroup: "unknown" diff --git a/manifests/zcloud-prod2/configmap.yaml b/manifests/zcloud-prod2/configmap.yaml index 4d02e63b1..983ce87d0 100644 --- a/manifests/zcloud-prod2/configmap.yaml +++ b/manifests/zcloud-prod2/configmap.yaml @@ -21,9 +21,10 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 - egressMetrics: + accessLogging: enabled: true - gcpProjectId: "zcloud-prod2" - cdfAggregationWindowSeconds: 60 - batchSize: 20 - flushIntervalSeconds: 15 + clientRegionHeader: "x-client-region" + clientRegionSubdivisionHeader: "x-client-region-subdivision" + clientIpHeader: "x-forwarded-for" + pricingGroups: {} + defaultPricingGroup: "unknown" diff --git a/manifests/zing-preview/configmap.yaml b/manifests/zing-preview/configmap.yaml index 7bb9a1ed6..983ce87d0 100644 --- a/manifests/zing-preview/configmap.yaml +++ b/manifests/zing-preview/configmap.yaml @@ -21,9 +21,10 @@ data: queryTablePageSizeLimit: 10000 queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 - egressMetrics: + accessLogging: enabled: true - gcpProjectId: "zing-preview" - cdfAggregationWindowSeconds: 60 - batchSize: 20 - flushIntervalSeconds: 15 + clientRegionHeader: "x-client-region" + clientRegionSubdivisionHeader: "x-client-region-subdivision" + clientIpHeader: "x-forwarded-for" + pricingGroups: {} + defaultPricingGroup: "unknown" diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 4c3653798..571bca0f4 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -17,9 +17,11 @@ package io.delta.sharing.server import java.io.{ByteArrayOutputStream, File, FileNotFoundException} +import java.net.InetAddress import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.AccessDeniedException import java.security.MessageDigest +import java.util.Locale import java.util.concurrent.CompletableFuture import javax.annotation.Nullable @@ -44,10 +46,10 @@ import org.slf4j.LoggerFactory import scalapb.json4s.Printer import io.delta.sharing.server.common.JsonUtils -import io.delta.sharing.server.config.ServerConfig +import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ -import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry, EgressMetricPoint, EgressMetricsEmitter} +import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry} object ErrorCode { val UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -191,14 +193,10 @@ class DeltaSharingService(serverConfig: ServerConfig) { private val deltaSharedTableLoader = new DeltaSharedTableLoader(serverConfig) private val logger = LoggerFactory.getLogger(classOf[DeltaSharingService]) - private val egressMetricsEmitter = EgressMetricsEmitter.create(serverConfig) private val accessLogEmitter = AccessLogEmitter.create(serverConfig) - private val cdfAggregationWindowMs = Option(serverConfig.getEgressMetrics) - .map(_.cdfAggregationWindowSeconds.toLong * 1000L) - .getOrElse(60000L) - private val cdfAggregates = scala.collection.mutable.HashMap.empty[(String, Long), Long] private def emitQueryEgressMetric( + req: HttpRequest, share: String, schema: String, table: String, @@ -209,6 +207,9 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() + val location = DeltaSharingService.buildClientLocationContext( + req, + serverConfig.getAccessLogging) // Emit to access log (new approach) val entry = AccessLogEntry( @@ -217,22 +218,19 @@ class DeltaSharingService(serverConfig: ServerConfig) { table = table, egressBytes = bytes, requestType = AccessLogEmitter.QueryRequestType, - timestampMs = nowMs + timestampMs = nowMs, + clientRegion = location.clientRegion, + clientRegionSubdivision = location.clientRegionSubdivision, + clientIp = location.clientIp, + clientIpSource = location.clientIpSource, + clientPricingGroup = location.clientPricingGroup, + clientLocationSource = location.clientLocationSource ) accessLogEmitter.record(entry) - - // Also emit to legacy metrics (deprecated, for backward compatibility during transition) - val point = EgressMetricPoint( - share = share, - requestType = EgressMetricsEmitter.QueryRequestType, - egressBytes = bytes, - startTimeMs = nowMs, - endTimeMs = nowMs - ) - egressMetricsEmitter.record(point) } private def emitCdfEgressMetric( + req: HttpRequest, share: String, schema: String, table: String, @@ -243,6 +241,9 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() + val location = DeltaSharingService.buildClientLocationContext( + req, + serverConfig.getAccessLogging) // Emit to access log immediately (new approach - no aggregation) val entry = AccessLogEntry( @@ -251,37 +252,15 @@ class DeltaSharingService(serverConfig: ServerConfig) { table = table, egressBytes = bytes, requestType = AccessLogEmitter.CdfStreamRequestType, - timestampMs = nowMs + timestampMs = nowMs, + clientRegion = location.clientRegion, + clientRegionSubdivision = location.clientRegionSubdivision, + clientIp = location.clientIp, + clientIpSource = location.clientIpSource, + clientPricingGroup = location.clientPricingGroup, + clientLocationSource = location.clientLocationSource ) accessLogEmitter.record(entry) - - // Also emit to legacy aggregated metrics (deprecated) - val windowStart = (nowMs / cdfAggregationWindowMs) * cdfAggregationWindowMs - - cdfAggregates.synchronized { - flushExpiredCdfAggregates(nowMs) - val key = (share, windowStart) - val current = cdfAggregates.getOrElse(key, 0L) - cdfAggregates.update(key, current + bytes) - } - } - - private def flushExpiredCdfAggregates(nowMs: Long): Unit = { - val expiredKeys = cdfAggregates.keys.filter { case (_, startMs) => - startMs + cdfAggregationWindowMs <= nowMs - }.toSeq - expiredKeys.foreach { key => - val (share, startMs) = key - val bytes = cdfAggregates.remove(key).get - val point = EgressMetricPoint( - share = share, - requestType = EgressMetricsEmitter.CdfStreamRequestType, - egressBytes = bytes, - startTimeMs = startMs, - endTimeMs = startMs + cdfAggregationWindowMs - ) - egressMetricsEmitter.record(point) - } } private def logCdfRequestComplete( @@ -569,7 +548,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { ) } - emitQueryEgressMetric(share, schema, table, queryResult.actions) + emitQueryEgressMetric(req, share, schema, table, queryResult.actions) streamingOutput(Some(queryResult.version), queryResult.responseFormat, queryResult.actions) } @@ -722,7 +701,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { ) } - emitQueryEgressMetric(share, schema, table, queryResult.actions) + emitQueryEgressMetric(req, share, schema, table, queryResult.actions) logTableQueryComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), @@ -781,7 +760,7 @@ class DeltaSharingService(serverConfig: ServerConfig) { deltaLogUpdateNs = updateNs, requestTimeoutSecondsForLogging = Some(serverConfig.requestTimeoutSeconds) ) - emitCdfEgressMetric(share, schema, table, queryResult.actions) + emitCdfEgressMetric(req, share, schema, table, queryResult.actions) logCdfRequestComplete(start, share, schema, table, queryResult) streamingOutput( Some(queryResult.version), @@ -837,11 +816,212 @@ object DeltaSharingService { val DELTA_SHARING_INCLUDE_END_STREAM_ACTION = "includeendstreamaction" val DELTA_SHARING_READER_FEATURES = "readerfeatures" val DELTA_SHARING_CAPABILITIES_DELIMITER = ";" + private val DefaultRegionHeaders = Seq( + "x-client-region", + "x-appengine-country", + "cf-ipcountry", + "cloudfront-viewer-country") + private val DefaultSubdivisionHeaders = Seq( + "x-client-region-subdivision", + "x-appengine-region") + private val DefaultIpHeaders = Seq( + "x-forwarded-for", + "x-envoy-external-address", + "x-real-ip", + "true-client-ip") + private val DefaultPricingGroupsByRegion = Map( + "US" -> "na_eu", + "CA" -> "na_eu", + "MX" -> "na_eu", + "GB" -> "na_eu", + "IE" -> "na_eu", + "DE" -> "na_eu", + "FR" -> "na_eu", + "NL" -> "na_eu", + "BE" -> "na_eu", + "CH" -> "na_eu", + "AT" -> "na_eu", + "ES" -> "na_eu", + "PT" -> "na_eu", + "IT" -> "na_eu", + "SE" -> "na_eu", + "NO" -> "na_eu", + "DK" -> "na_eu", + "FI" -> "na_eu", + "PL" -> "na_eu", + "CZ" -> "na_eu", + "HU" -> "na_eu", + "RO" -> "na_eu", + "JP" -> "apac", + "KR" -> "apac", + "IN" -> "apac", + "SG" -> "apac", + "HK" -> "apac", + "TW" -> "apac", + "ID" -> "apac", + "MY" -> "apac", + "PH" -> "apac", + "TH" -> "apac", + "VN" -> "apac", + "AU" -> "apac", + "NZ" -> "apac", + "BR" -> "latam", + "AR" -> "latam", + "CL" -> "latam", + "CO" -> "latam", + "PE" -> "latam") + + private[server] case class ClientLocationContext( + clientRegion: Option[String], + clientRegionSubdivision: Option[String], + clientIp: Option[String], + clientIpSource: Option[String], + clientPricingGroup: String, + clientLocationSource: Option[String]) private[server] def extractEgressBytes(actions: Seq[Object]): Long = { actions.map(extractActionBytes).sum } + private[server] def buildClientLocationContext( + req: HttpRequest, + accessLoggingConfig: AccessLoggingConfig): ClientLocationContext = { + val headerMap: Map[String, String] = req.headers().names().asScala + .flatMap { name => + Option(req.headers().get(name)).map { value => + name.toString.toLowerCase(Locale.ROOT) -> value + } + } + .toMap + buildClientLocationContext(headerMap, accessLoggingConfig) + } + + private[server] def buildClientLocationContext( + requestHeaders: Map[String, String], + accessLoggingConfig: AccessLoggingConfig): ClientLocationContext = { + val cfgOpt = Option(accessLoggingConfig) + val normalizedHeaders = requestHeaders.map { case (k, v) => + k.toLowerCase(Locale.ROOT) -> v + } + + val regionHeaders = getOrderedHeaders( + cfgOpt.flatMap(c => Option(c.getClientRegionHeader)), + DefaultRegionHeaders) + val subdivisionHeaders = getOrderedHeaders( + cfgOpt.flatMap(c => Option(c.getClientRegionSubdivisionHeader)), + DefaultSubdivisionHeaders) + val ipHeaders = getOrderedHeaders( + cfgOpt.flatMap(c => Option(c.getClientIpHeader)), + DefaultIpHeaders) + + val regionWithSource = getFirstLocation(regionHeaders, normalizedHeaders) + val subdivisionWithSource = getFirstLocation(subdivisionHeaders, normalizedHeaders) + val ipWithSource = getFirstClientIp(ipHeaders, normalizedHeaders) + + val region = regionWithSource.map(_._1) + val subdivision = subdivisionWithSource.map(_._1) + val clientIp = ipWithSource.map(_._1) + val clientIpSource = ipWithSource.map(_._2) + val source = regionWithSource.map(_._2).orElse(subdivisionWithSource.map(_._2)) + + val configuredPricingMap = cfgOpt + .flatMap(c => Option(c.getPricingGroups)) + .map(_.asScala.toMap) + .getOrElse(Map.empty[String, String]) + .map { case (k, v) => normalizeLocationCode(k) -> v.trim } + .filter { case (k, v) => k.nonEmpty && v.nonEmpty } + + val pricingMap = if (configuredPricingMap.nonEmpty) { + configuredPricingMap + } else { + DefaultPricingGroupsByRegion + } + + val defaultPricingGroup = cfgOpt + .flatMap(c => Option(c.getDefaultPricingGroup)) + .map(_.trim) + .filter(_.nonEmpty) + .getOrElse("unknown") + + val pricingGroup = subdivision.flatMap(pricingMap.get) + .orElse(region.flatMap(pricingMap.get)) + .orElse(pricingMap.get("*")) + .orElse(region) + .getOrElse(defaultPricingGroup) + + ClientLocationContext( + clientRegion = region, + clientRegionSubdivision = subdivision, + clientIp = clientIp, + clientIpSource = clientIpSource, + clientPricingGroup = pricingGroup, + clientLocationSource = source) + } + + private def getOrderedHeaders( + configuredHeader: Option[String], + defaultHeaders: Seq[String]): Seq[String] = { + (configuredHeader.toSeq ++ defaultHeaders) + .map(_.trim.toLowerCase(Locale.ROOT)) + .filter(_.nonEmpty) + .distinct + } + + private def getFirstLocation( + headers: Seq[String], + requestHeaders: Map[String, String]): Option[(String, String)] = { + headers.iterator + .flatMap { h => + requestHeaders.get(h) + .map(v => normalizeLocationCode(v) -> h) + } + .find(_._1.nonEmpty) + } + + private def getFirstClientIp( + headers: Seq[String], + requestHeaders: Map[String, String]): Option[(String, String)] = { + headers.iterator + .flatMap { h => + requestHeaders.get(h) + .flatMap(v => extractClientIp(v).map(_ -> h)) + } + .toSeq + .headOption + } + + private def extractClientIp(value: String): Option[String] = { + val candidates = value.split(",").map(_.trim).filter(_.nonEmpty) + val valid = candidates.filter(isValidIp) + valid.find(isPublicIp).orElse(valid.headOption) + } + + private def isValidIp(ip: String): Boolean = { + try { + InetAddress.getByName(ip) + true + } catch { + case _: Throwable => false + } + } + + private def isPublicIp(ip: String): Boolean = { + try { + val inet = InetAddress.getByName(ip) + !inet.isAnyLocalAddress && + !inet.isLoopbackAddress && + !inet.isLinkLocalAddress && + !inet.isSiteLocalAddress && + !inet.isMulticastAddress + } catch { + case _: Throwable => false + } + } + + private def normalizeLocationCode(value: String): String = { + value.trim.toUpperCase(Locale.ROOT).replaceAll("[^A-Z0-9*]", "") + } + private def extractActionBytes(action: Object): Long = { action match { case singleAction: SingleAction => diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index b88e62d7a..5acd545b2 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -68,8 +68,6 @@ case class ServerConfig( @BeanProperty var refreshTokenTtlMs: Int, // Whether to emit performance/timing log lines for table queries and CDF requests. @BeanProperty var perfLoggingEnabled: Boolean, - // Telemetry settings for egress metrics (deprecated, use accessLogging). - @BeanProperty var egressMetrics: EgressMetricsConfig, // Access logging configuration for tracking share data egress via structured logs. @BeanProperty var accessLogging: AccessLoggingConfig ) extends ConfigItem { @@ -96,7 +94,6 @@ case class ServerConfig( queryTablePageTokenTtlMs = 259200000, // 3 days refreshTokenTtlMs = 3600000, // 1 hour perfLoggingEnabled = true, - egressMetrics = null, accessLogging = null ) } @@ -128,63 +125,46 @@ case class ServerConfig( if (ssl != null) { ssl.checkConfig() } - if (egressMetrics != null) { - egressMetrics.checkConfig() - } if (accessLogging != null) { accessLogging.checkConfig() } } } -case class EgressMetricsConfig( - @BeanProperty var enabled: Boolean, - @BeanProperty var gcpProjectId: String, - @BeanProperty var cdfAggregationWindowSeconds: Int, - @BeanProperty var batchSize: Int, - @BeanProperty var flushIntervalSeconds: Int) extends ConfigItem { - - def this() = { - this( - enabled = false, - gcpProjectId = null, - cdfAggregationWindowSeconds = 60, - batchSize = 20, - flushIntervalSeconds = 15 - ) - } - - override def checkConfig(): Unit = { - if (enabled && (gcpProjectId == null || gcpProjectId.trim.isEmpty)) { - throw new IllegalArgumentException("'gcpProjectId' in 'egressMetrics' must be provided") - } - if (cdfAggregationWindowSeconds <= 0) { - throw new IllegalArgumentException( - "'cdfAggregationWindowSeconds' in 'egressMetrics' must be greater than 0") - } - if (batchSize <= 0) { - throw new IllegalArgumentException("'batchSize' in 'egressMetrics' must be greater than 0") - } - if (flushIntervalSeconds <= 0) { - throw new IllegalArgumentException( - "'flushIntervalSeconds' in 'egressMetrics' must be greater than 0") - } - } -} - /** * Configuration for access logging to track share data egress. * When enabled, structured JSON logs are emitted for each data access. */ case class AccessLoggingConfig( - @BeanProperty var enabled: Boolean) extends ConfigItem { + @BeanProperty var enabled: Boolean, + // Header that contains the client region code (for example: US, DE). + @BeanProperty var clientRegionHeader: String, + // Header that contains the client region subdivision (for example: USCA, DEBE). + @BeanProperty var clientRegionSubdivisionHeader: String, + // Header that contains client IP or forwarding chain (for example: X-Forwarded-For). + @BeanProperty var clientIpHeader: String, + // Optional mapping from region/subdivision codes to pricing group labels. + // Keys should be uppercase location codes (for example: US, DE, USCA). + // A wildcard key "*" can be used as a catch-all default. + @BeanProperty var pricingGroups: java.util.Map[String, String], + // Fallback group when no mapping and no region are available. + @BeanProperty var defaultPricingGroup: String) extends ConfigItem { def this() = { - this(enabled = false) + this( + enabled = false, + clientRegionHeader = "x-client-region", + clientRegionSubdivisionHeader = "x-client-region-subdivision", + clientIpHeader = "x-forwarded-for", + pricingGroups = Collections.emptyMap(), + defaultPricingGroup = "unknown") } override def checkConfig(): Unit = { - // No additional validation needed - enabled is a simple boolean + if (defaultPricingGroup == null || defaultPricingGroup.trim.isEmpty) { + throw new IllegalArgumentException( + "'defaultPricingGroup' in 'accessLogging' must be provided") + } } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index ee6984c82..791649aa0 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -30,6 +30,12 @@ import io.delta.sharing.server.config.ServerConfig * @param egressBytes The number of bytes transferred * @param requestType The type of request: "query" or "cdf_stream" * @param timestampMs The timestamp of the access in milliseconds since epoch + * @param clientRegion Optional client region code derived from request headers (for example: US) + * @param clientRegionSubdivision Optional client subdivision code (for example: USCA) + * @param clientIp Optional client IP derived from forwarding headers + * @param clientIpSource Optional header name used to resolve client IP + * @param clientPricingGroup Pricing group label used for downstream egress cost attribution + * @param clientLocationSource Optional header name used to resolve client location */ case class AccessLogEntry( share: String, @@ -37,7 +43,13 @@ case class AccessLogEntry( table: String, egressBytes: Long, requestType: String, - timestampMs: Long) + timestampMs: Long, + clientRegion: Option[String] = None, + clientRegionSubdivision: Option[String] = None, + clientIp: Option[String] = None, + clientIpSource: Option[String] = None, + clientPricingGroup: String = "unknown", + clientLocationSource: Option[String] = None) /** * Trait for emitting access logs for share data egress tracking. @@ -84,16 +96,27 @@ class JsonAccessLogEmitter extends AccessLogEmitter { return } - val logPayload = Map( + val basePayload = Map( "logType" -> "ACCESS_LOG", "share" -> entry.share, "schema" -> entry.schema, "table" -> entry.table, "egressBytes" -> entry.egressBytes, "requestType" -> entry.requestType, - "timestampMs" -> entry.timestampMs + "timestampMs" -> entry.timestampMs, + "clientPricingGroup" -> entry.clientPricingGroup ) + val locationPayload = Seq( + entry.clientRegion.map("clientRegion" -> _), + entry.clientRegionSubdivision.map("clientRegionSubdivision" -> _), + entry.clientIp.map("clientIp" -> _), + entry.clientIpSource.map("clientIpSource" -> _), + entry.clientLocationSource.map("clientLocationSource" -> _) + ).flatten.toMap + + val logPayload = basePayload ++ locationPayload + logger.info(JsonUtils.toJson(logPayload)) } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala deleted file mode 100644 index 289ea7476..000000000 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitter.scala +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (2021) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.sharing.server.telemetry - -import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStreamWriter} -import java.net.{HttpURLConnection, URL} -import java.nio.charset.StandardCharsets.UTF_8 -import java.util.Collections -import java.util.concurrent.{Executors, ScheduledExecutorService, ThreadFactory, TimeUnit} -import java.util.concurrent.atomic.AtomicBoolean - -import scala.collection.mutable.ArrayBuffer -import scala.util.Random - -import com.google.auth.oauth2.GoogleCredentials -import org.slf4j.LoggerFactory - -import io.delta.sharing.server.common.JsonUtils -import io.delta.sharing.server.config.{EgressMetricsConfig, ServerConfig} - -case class EgressMetricPoint( - share: String, - requestType: String, - egressBytes: Long, - startTimeMs: Long, - endTimeMs: Long) - -trait EgressMetricsEmitter { - def record(point: EgressMetricPoint): Unit -} - -case class MonitoringWriteResult(status: Int, responseBody: Option[String] = None) - -trait MonitoringTimeSeriesClient { - def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult -} - -private class HttpMonitoringTimeSeriesClient(config: EgressMetricsConfig) - extends MonitoringTimeSeriesClient { - - private val scope = Collections.singletonList("https://www.googleapis.com/auth/monitoring.write") - private val metricWriteUrl = - s"https://monitoring.googleapis.com/v3/projects/${config.gcpProjectId}/timeSeries" - private var credentials: GoogleCredentials = _ - - override def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult = { - val body = JsonUtils.toJson(Map("timeSeries" -> timeSeries)) - val connection = new URL(metricWriteUrl).openConnection().asInstanceOf[HttpURLConnection] - try { - connection.setRequestMethod("POST") - connection.setConnectTimeout(3000) - connection.setReadTimeout(5000) - connection.setDoOutput(true) - connection.setRequestProperty("Authorization", s"Bearer ${accessTokenValue}") - connection.setRequestProperty("Content-Type", "application/json; charset=utf-8") - - val writer = new OutputStreamWriter(connection.getOutputStream, UTF_8) - try { - writer.write(body) - } finally { - writer.close() - } - val status = connection.getResponseCode - MonitoringWriteResult(status, readResponseBody(connection, status)) - } finally { - connection.disconnect() - } - } - - private def readResponseBody(connection: HttpURLConnection, status: Int): Option[String] = { - val stream: InputStream = - if (status / 100 == 2) { - null - } else { - val err = connection.getErrorStream - if (err != null) err else connection.getInputStream - } - if (stream == null) { - None - } else { - val reader = new BufferedReader(new InputStreamReader(stream, UTF_8)) - try { - val sb = new StringBuilder - var line = reader.readLine() - while (line != null) { - if (sb.nonEmpty) sb.append('\n') - sb.append(line) - line = reader.readLine() - } - val body = sb.toString().trim - if (body.isEmpty) None else Some(body) - } finally { - reader.close() - } - } - } - - private def accessTokenValue: String = synchronized { - if (credentials == null) { - credentials = GoogleCredentials.getApplicationDefault().createScoped(scope) - } - val token = credentials.getAccessToken - if (token == null || - token.getExpirationTime == null || - token.getExpirationTime.getTime <= (System.currentTimeMillis() + 60000L)) { - credentials.refresh() - } - credentials.getAccessToken.getTokenValue - } -} - -object EgressMetricsEmitter { - val ShareEgressBytesMetric = "custom.googleapis.com/delta_sharing/share_egress_bytes" - val UnknownShare = "unknown" - val QueryRequestType = "query" - val CdfStreamRequestType = "cdf_stream" - - def create(serverConfig: ServerConfig): EgressMetricsEmitter = { - val cfg = Option(serverConfig.getEgressMetrics) - cfg match { - case Some(c) if c.enabled => new GcpCloudMonitoringEgressMetricsEmitter(c) - case _ => NoopEgressMetricsEmitter - } - } -} - -object NoopEgressMetricsEmitter extends EgressMetricsEmitter { - override def record(point: EgressMetricPoint): Unit = {} -} - -private case class CumulativeTotals(var bytes: Long) - -private sealed trait SendResult -private case object SendSuccess extends SendResult -private case object SendRetryableFailure extends SendResult -private case class SendNonRetryableFailure(status: Int, responseBody: Option[String]) - extends SendResult - -class GcpCloudMonitoringEgressMetricsEmitter( - config: EgressMetricsConfig, - client: MonitoringTimeSeriesClient, - scheduler: ScheduledExecutorService, - registerShutdownHook: Boolean, - maxPendingSeries: Int, - maxRetryAttempts: Int, - retryBaseDelayMs: Long) extends EgressMetricsEmitter { - - def this(config: EgressMetricsConfig) = { - this( - config = config, - client = new HttpMonitoringTimeSeriesClient(config), - scheduler = GcpCloudMonitoringEgressMetricsEmitter.newDefaultScheduler(), - registerShutdownHook = true, - maxPendingSeries = math.max(config.batchSize * 200, config.batchSize), - maxRetryAttempts = 3, - retryBaseDelayMs = 200L - ) - } - - import EgressMetricsEmitter._ - - private val logger = LoggerFactory.getLogger(classOf[GcpCloudMonitoringEgressMetricsEmitter]) - private val cumulativeByLabel = - scala.collection.mutable.HashMap.empty[String, CumulativeTotals] - private val pendingSeries = ArrayBuffer.empty[Map[String, Any]] - private val flushInFlight = new AtomicBoolean(false) - private val random = new Random() - private var droppedSeriesCount = 0L - - scheduler.scheduleAtFixedRate( - new Runnable { - override def run(): Unit = { - triggerAsyncFlush() - } - }, - config.flushIntervalSeconds, - config.flushIntervalSeconds, - TimeUnit.SECONDS) - - if (registerShutdownHook) { - // scalastyle:off runtimeaddshutdownhook - Runtime.getRuntime.addShutdownHook(new Thread(new Runnable { - override def run(): Unit = { - try { - flushPendingSynchronously() - } finally { - scheduler.shutdownNow() - } - } - })) - // scalastyle:on runtimeaddshutdownhook - } - - override def record(point: EgressMetricPoint): Unit = synchronized { - val share = sanitizeLabel(point.share, UnknownShare) - val requestType = sanitizeLabel(point.requestType, QueryRequestType) - val key = s"$share|$requestType" - val totals = cumulativeByLabel.getOrElseUpdate(key, CumulativeTotals(0L)) - totals.bytes += point.egressBytes - val endMs = math.max(point.endTimeMs, System.currentTimeMillis()) - - pendingSeries += makeTimeSeries( - ShareEgressBytesMetric, - share, - requestType, - endMs, - Map("int64Value" -> totals.bytes.toString)) - - if (pendingSeries.size > maxPendingSeries) { - val overflow = pendingSeries.size - maxPendingSeries - pendingSeries.remove(0, overflow) - droppedSeriesCount += overflow - if (droppedSeriesCount % 100 == 0) { - logger.warn( - s"Dropped $droppedSeriesCount queued egress metric points due to queue overload") - } - } - - if (pendingSeries.size >= config.batchSize) { - triggerAsyncFlush() - } - } - - private def triggerAsyncFlush(): Unit = { - if (flushInFlight.compareAndSet(false, true)) { - scheduler.execute(new Runnable { - override def run(): Unit = { - try { - flushPendingSynchronously() - } finally { - flushInFlight.set(false) - } - } - }) - } - } - - private[telemetry] def snapshotPendingSeriesSize(): Int = synchronized { - pendingSeries.size - } - - private def sanitizeLabel(value: String, fallback: String): String = { - if (value == null || value.trim.isEmpty) fallback else value.trim - } - - private def toRfc3339(ms: Long): String = { - java.time.Instant.ofEpochMilli(ms).toString - } - - private def makeTimeSeries( - metricType: String, - share: String, - requestType: String, - endTimeMs: Long, - value: Map[String, Any]): Map[String, Any] = { - Map( - "metric" -> Map( - "type" -> metricType, - "labels" -> Map( - "share" -> share, - "request_type" -> requestType - ) - ), - "resource" -> Map( - "type" -> "global", - "labels" -> Map("project_id" -> config.gcpProjectId) - ), - "points" -> Seq(Map( - "interval" -> Map( - // Cloud Monitoring treats this custom metric as gauge, requiring point start==end. - "startTime" -> toRfc3339(endTimeMs), - "endTime" -> toRfc3339(endTimeMs) - ), - "value" -> value - )) - ) - } - - private def flushPendingSynchronously(): Unit = { - val snapshot = synchronized { - if (pendingSeries.isEmpty) { - Seq.empty[Map[String, Any]] - } else { - val batch = pendingSeries.toVector - pendingSeries.clear() - batch - } - } - if (snapshot.isEmpty) { - return - } - - val batches = snapshot.grouped(config.batchSize).toSeq - batches.foreach { batch => - handleBatchWrite(batch) - } - } - - private def handleBatchWrite(batch: Seq[Map[String, Any]]): Unit = { - sendWithRetries(batch) match { - case SendSuccess => - case SendRetryableFailure => - synchronized { - pendingSeries.prependAll(batch) - } - case SendNonRetryableFailure(status, responseBody) if batch.size > 1 => - val (left, right) = batch.splitAt(batch.size / 2) - handleBatchWrite(left) - handleBatchWrite(right) - case SendNonRetryableFailure(status, responseBody) => - logger.warn( - s"Dropping non-retryable egress metric point. status=$status" + - formatResponseBodyForLog(responseBody)) - } - } - - private def sendWithRetries(batch: Seq[Map[String, Any]]): SendResult = { - var attempt = 1 - while (attempt <= maxRetryAttempts) { - try { - val writeResult = client.write(batch) - val status = writeResult.status - if (status / 100 == 2) { - return SendSuccess - } - if (!isRetryableStatus(status)) { - return SendNonRetryableFailure(status, writeResult.responseBody) - } - logger.warn( - s"Retryable egress metrics write failure status=$status " + - s"(attempt=$attempt/$maxRetryAttempts)" + - formatResponseBodyForLog(writeResult.responseBody)) - if (attempt < maxRetryAttempts) { - backoffSleep(attempt) - } - } catch { - case e: Throwable => - logger.warn( - s"Failed to write egress metrics (attempt=$attempt/$maxRetryAttempts)", - e) - if (attempt < maxRetryAttempts) { - backoffSleep(attempt) - } - } - attempt += 1 - } - SendRetryableFailure - } - - private def isRetryableStatus(status: Int): Boolean = { - status == 429 || status == 500 || status == 502 || status == 503 || status == 504 - } - - private def backoffSleep(attempt: Int): Unit = { - val jitter = random.nextInt(100) - val delay = retryBaseDelayMs * (1L << (attempt - 1)) + jitter - Thread.sleep(delay) - } - - private def formatResponseBodyForLog(responseBody: Option[String]): String = { - responseBody match { - case Some(body) if body.nonEmpty => - val maxLen = 2000 - val normalized = body.replaceAll("\\s+", " ").trim - val truncated = if (normalized.length <= maxLen) normalized else { - normalized.substring(0, maxLen) + "..." - } - s" responseBody=$truncated" - case _ => "" - } - } -} - -object GcpCloudMonitoringEgressMetricsEmitter { - private[telemetry] def newDefaultScheduler(): ScheduledExecutorService = { - Executors.newSingleThreadScheduledExecutor(new ThreadFactory { - override def newThread(r: Runnable): Thread = { - val t = new Thread(r, "delta-sharing-egress-metrics") - t.setDaemon(true) - t - } - }) - } -} diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index 5eb8aaabf..a2e7f550b 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -23,6 +23,7 @@ import java.security.cert.X509Certificate import java.sql.Timestamp import javax.net.ssl._ +import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import com.linecorp.armeria.server.Server @@ -34,7 +35,7 @@ import scalapb.json4s.JsonFormat import io.delta.sharing.server.DeltaSharingService.DELTA_SHARING_INCLUDE_END_STREAM_ACTION import io.delta.sharing.server.common.JsonUtils import io.delta.sharing.server.common.actions.{ColumnMappingTableFeature, DeletionVectorDescriptor, DeletionVectorsTableFeature, DeltaAddFile, DeltaFormat, DeltaProtocol, DeltaSingleAction} -import io.delta.sharing.server.config.ServerConfig +import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model._ import io.delta.sharing.server.protocol._ @@ -270,6 +271,80 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { assert(DeltaSharingService.extractEgressBytes(actions) == 45L) } + test("buildClientLocationContext uses configured geo headers and pricing map") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setPricingGroups(Map("US" -> "na-tier-1", "USCA" -> "us-west").asJava) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map( + "X-Client-Region" -> "us", + "X-Client-Region-Subdivision" -> "us-ca" + ), + cfg) + + assert(ctx.clientRegion.contains("US")) + assert(ctx.clientRegionSubdivision.contains("USCA")) + assert(ctx.clientIp.isEmpty) + assert(ctx.clientPricingGroup == "us-west") + assert(ctx.clientLocationSource.contains("x-client-region")) + } + + test("buildClientLocationContext falls back to default headers and wildcard pricing") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setPricingGroups(Map("*" -> "global-tier-2").asJava) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map("CF-IPCountry" -> "br"), + cfg) + + assert(ctx.clientRegion.contains("BR")) + assert(ctx.clientRegionSubdivision.isEmpty) + assert(ctx.clientIp.isEmpty) + assert(ctx.clientPricingGroup == "global-tier-2") + assert(ctx.clientLocationSource.contains("cf-ipcountry")) + } + + test("buildClientLocationContext uses fallback pricing group when location is unavailable") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setDefaultPricingGroup("unknown-tier") + + val ctx = DeltaSharingService.buildClientLocationContext(Map.empty[String, String], cfg) + + assert(ctx.clientRegion.isEmpty) + assert(ctx.clientRegionSubdivision.isEmpty) + assert(ctx.clientIp.isEmpty) + assert(ctx.clientPricingGroup == "unknown-tier") + assert(ctx.clientLocationSource.isEmpty) + } + + test("buildClientLocationContext extracts client IP from forwarding headers") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map("X-Forwarded-For" -> "10.0.0.1, 217.9.4.27, 35.191.144.206"), + cfg) + + assert(ctx.clientRegion.isEmpty) + assert(ctx.clientIp.contains("217.9.4.27")) + assert(ctx.clientIpSource.contains("x-forwarded-for")) + } + + test("buildClientLocationContext applies default pricing groups when no custom map") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map("X-Appengine-Country" -> "de"), + cfg) + + assert(ctx.clientRegion.contains("DE")) + assert(ctx.clientPricingGroup == "na_eu") + } + integrationTest("401 Unauthorized Error: incorrect token") { val url = requestPath("/shares") val connection = new URL(url).openConnection().asInstanceOf[HttpsURLConnection] diff --git a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala index 7e75385b8..2258cd0e6 100644 --- a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala @@ -20,6 +20,7 @@ import java.io.File import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.Files import java.util.Arrays +import java.util.Collections import org.apache.commons.io.FileUtils import org.scalatest.FunSuite @@ -98,20 +99,16 @@ class ServerConfigSuite extends FunSuite { serverConfig.setVersion(1) serverConfig.setShares(sharesInTemplate) serverConfig.setPort(8080) - val egressMetrics = new EgressMetricsConfig() - egressMetrics.setEnabled(false) - egressMetrics.setGcpProjectId("") - egressMetrics.setCdfAggregationWindowSeconds(60) - egressMetrics.setBatchSize(20) - egressMetrics.setFlushIntervalSeconds(15) - serverConfig.setEgressMetrics(egressMetrics) + val accessLogging = new AccessLoggingConfig() + accessLogging.setEnabled(false) + serverConfig.setAccessLogging(accessLogging) assert(loaded == serverConfig) } finally { tempFile.delete() } } - test("egress metrics config") { + test("access logging config") { val serverConfig = new ServerConfig() serverConfig.setVersion(1) serverConfig.setShares(Arrays.asList( @@ -125,13 +122,14 @@ class ServerConfigSuite extends FunSuite { )) )) )) - val egressMetrics = new EgressMetricsConfig() - egressMetrics.setEnabled(true) - egressMetrics.setGcpProjectId("project-1") - egressMetrics.setCdfAggregationWindowSeconds(60) - egressMetrics.setBatchSize(10) - egressMetrics.setFlushIntervalSeconds(5) - serverConfig.setEgressMetrics(egressMetrics) + val accessLogging = new AccessLoggingConfig() + accessLogging.setEnabled(true) + accessLogging.setClientRegionHeader("x-client-region") + accessLogging.setClientRegionSubdivisionHeader("x-client-region-subdivision") + accessLogging.setClientIpHeader("x-forwarded-for") + accessLogging.setPricingGroups(Collections.singletonMap("US", "na-tier-1")) + accessLogging.setDefaultPricingGroup("unknown") + serverConfig.setAccessLogging(accessLogging) testConfig( """version: 1 |shares: @@ -141,12 +139,14 @@ class ServerConfigSuite extends FunSuite { | tables: | - name: table1 | location: s3a://bucket/path - |egressMetrics: + |accessLogging: | enabled: true - | gcpProjectId: project-1 - | cdfAggregationWindowSeconds: 60 - | batchSize: 10 - | flushIntervalSeconds: 5 + | clientRegionHeader: x-client-region + | clientRegionSubdivisionHeader: x-client-region-subdivision + | clientIpHeader: x-forwarded-for + | pricingGroups: + | US: na-tier-1 + | defaultPricingGroup: unknown |""".stripMargin, serverConfig) } @@ -205,11 +205,14 @@ class ServerConfigSuite extends FunSuite { } } - test("EgressMetricsConfig") { - assertInvalidConfig("'gcpProjectId' in 'egressMetrics' must be provided") { - val e = new EgressMetricsConfig() - e.setEnabled(true) - e.checkConfig() + test("AccessLoggingConfig") { + val accessLogging = new AccessLoggingConfig() + accessLogging.setEnabled(true) + accessLogging.checkConfig() + + accessLogging.setDefaultPricingGroup(" ") + assertInvalidConfig("'defaultPricingGroup' in 'accessLogging' must be provided") { + accessLogging.checkConfig() } } diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala index 2563e8c97..f4c28ecf1 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala @@ -29,7 +29,13 @@ class AccessLogEmitterSuite extends FunSuite { table = "test-table", egressBytes = 1024L, requestType = AccessLogEmitter.QueryRequestType, - timestampMs = System.currentTimeMillis() + timestampMs = System.currentTimeMillis(), + clientRegion = Some("US"), + clientRegionSubdivision = Some("USCA"), + clientIp = Some("217.9.4.27"), + clientIpSource = Some("x-forwarded-for"), + clientPricingGroup = "na-tier-1", + clientLocationSource = Some("x-client-region") ) assert(entry.share == "test-share") @@ -37,6 +43,12 @@ class AccessLogEmitterSuite extends FunSuite { assert(entry.table == "test-table") assert(entry.egressBytes == 1024L) assert(entry.requestType == "query") + assert(entry.clientRegion.contains("US")) + assert(entry.clientRegionSubdivision.contains("USCA")) + assert(entry.clientIp.contains("217.9.4.27")) + assert(entry.clientIpSource.contains("x-forwarded-for")) + assert(entry.clientPricingGroup == "na-tier-1") + assert(entry.clientLocationSource.contains("x-client-region")) } test("NoopAccessLogEmitter ignores all records") { diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala deleted file mode 100644 index 4c84d2067..000000000 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/EgressMetricsEmitterSuite.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (2021) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.sharing.server.telemetry - -import java.util.concurrent.{Executors, ScheduledExecutorService} - -import org.scalatest.FunSuite - -import io.delta.sharing.server.config.EgressMetricsConfig - -class EgressMetricsEmitterSuite extends FunSuite { - - private class FakeClient(sleepMs: Long = 0L) extends MonitoringTimeSeriesClient { - val writes = scala.collection.mutable.ArrayBuffer.empty[Seq[Map[String, Any]]] - - override def write(timeSeries: Seq[Map[String, Any]]): MonitoringWriteResult = { - if (sleepMs > 0) { - Thread.sleep(sleepMs) - } - writes.synchronized { - writes += timeSeries - } - MonitoringWriteResult(200) - } - - def writeCount: Int = writes.synchronized { - writes.size - } - } - - private def mkConfig(batchSize: Int, flushIntervalSeconds: Int): EgressMetricsConfig = { - val cfg = new EgressMetricsConfig() - cfg.setEnabled(true) - cfg.setGcpProjectId("test-project") - cfg.setBatchSize(batchSize) - cfg.setFlushIntervalSeconds(flushIntervalSeconds) - cfg.setCdfAggregationWindowSeconds(60) - cfg - } - - private def waitUntil(condition: => Boolean, timeoutMs: Long): Unit = { - val deadline = System.currentTimeMillis() + timeoutMs - while (!condition && System.currentTimeMillis() < deadline) { - Thread.sleep(20) - } - assert(condition) - } - - private def newEmitter( - config: EgressMetricsConfig, - client: MonitoringTimeSeriesClient, - scheduler: ScheduledExecutorService): GcpCloudMonitoringEgressMetricsEmitter = { - new GcpCloudMonitoringEgressMetricsEmitter( - config = config, - client = client, - scheduler = scheduler, - registerShutdownHook = false, - maxPendingSeries = 1000, - maxRetryAttempts = 2, - retryBaseDelayMs = 10L) - } - - private def extractIntervals(client: FakeClient): Seq[(String, String)] = { - client.writes.synchronized { - client.writes.flatMap { batch => - batch.map { timeSeries => - val points = timeSeries("points").asInstanceOf[Seq[Map[String, Any]]] - val interval = points.head("interval").asInstanceOf[Map[String, Any]] - ( - interval("startTime").asInstanceOf[String], - interval("endTime").asInstanceOf[String] - ) - } - }.toSeq - } - } - - test("gauge metric points use equal start and end times") { - val scheduler = Executors.newSingleThreadScheduledExecutor() - try { - val client = new FakeClient() - val emitter = newEmitter(mkConfig(batchSize = 1, flushIntervalSeconds = 60), client, scheduler) - - emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) - emitter.record(EgressMetricPoint("share-a", "query", 20L, 7000L, 8000L)) - - waitUntil(client.writeCount >= 2, timeoutMs = 3000) - val intervals = extractIntervals(client) - assert(intervals.nonEmpty) - assert(intervals.forall { case (start, end) => start == end }) - } finally { - scheduler.shutdownNow() - } - } - - test("low traffic still flushes via scheduled flush") { - val scheduler = Executors.newSingleThreadScheduledExecutor() - try { - val client = new FakeClient() - val emitter = newEmitter(mkConfig(batchSize = 10, flushIntervalSeconds = 1), client, scheduler) - - emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) - - waitUntil(client.writeCount >= 1, timeoutMs = 4000) - } finally { - scheduler.shutdownNow() - } - } - - test("record path is non-blocking even when monitoring write is slow") { - val scheduler = Executors.newSingleThreadScheduledExecutor() - try { - val slowClient = new FakeClient(sleepMs = 500) - val emitter = newEmitter( - mkConfig(batchSize = 1, flushIntervalSeconds = 60), - slowClient, - scheduler) - - val startNs = System.nanoTime() - emitter.record(EgressMetricPoint("share-a", "query", 10L, 1000L, 2000L)) - val elapsedMs = (System.nanoTime() - startNs) / 1000000L - - assert(elapsedMs < 250L) - waitUntil(slowClient.writeCount >= 1, timeoutMs = 4000) - } finally { - scheduler.shutdownNow() - } - } -} diff --git a/server/src/universal/conf/delta-sharing-server.yaml.template b/server/src/universal/conf/delta-sharing-server.yaml.template index e30c0dd51..73d774aa3 100644 --- a/server/src/universal/conf/delta-sharing-server.yaml.template +++ b/server/src/universal/conf/delta-sharing-server.yaml.template @@ -67,14 +67,22 @@ queryTablePageTokenTtlMs: 259200000 refreshTokenTtlMs: 3600000 # Whether to emit performance/timing log lines for table queries and CDF requests. perfLoggingEnabled: true -# Optional share-attributed egress metrics emitted to Cloud Monitoring. -egressMetrics: +# Optional share-attributed egress access logs emitted as structured JSON log lines. +accessLogging: enabled: false - # Required when enabled=true. - gcpProjectId: "" - # CDF egress bytes are aggregated into this window before emission. - cdfAggregationWindowSeconds: 60 - # Max number of time series rows queued before flushing. - batchSize: 20 - # Flush cadence for queued time series writes. - flushIntervalSeconds: 15 + # Name of the incoming header that carries client region code (for example: US, DE). + # In GCP, configure your load balancer backend custom request headers to inject this. + # Example: X-Client-Region:{client_region} + clientRegionHeader: "x-client-region" + # Name of the incoming header that carries client subdivision (for example: USCA). + # Example: X-Client-Region-Subdivision:{client_region_subdivision} + clientRegionSubdivisionHeader: "x-client-region-subdivision" + # Name of the incoming header that carries client IP information. + # This is typically a forwarding chain header and does not require LB changes when already present. + clientIpHeader: "x-forwarded-for" + # Optional mapping from region/subdivision codes to billing price groups. + # Use uppercase keys. A wildcard key "*" can be used as a catch-all. + # If empty, server applies a built-in default map (na_eu/apac/latam buckets). + pricingGroups: {} + # Fallback pricing group when no location can be resolved. + defaultPricingGroup: "unknown" From 20e25fa47d32b3773096edfc11c6d3aeabb675d0 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Fri, 29 May 2026 11:24:09 +0300 Subject: [PATCH 06/15] Log headers --- .../io/delta/sharing/server/DeltaSharingService.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 571bca0f4..407bbe984 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -808,6 +808,9 @@ class DeltaSharingService(serverConfig: ServerConfig) { object DeltaSharingService { + // TEMPORARY: Logger for debugging headers - remove after testing X-Client-Region + private val logger = LoggerFactory.getLogger("delta.sharing.headers.debug") + val DELTA_TABLE_VERSION_HEADER = "Delta-Table-Version" val DELTA_TABLE_METADATA_CONTENT_TYPE = "application/x-ndjson; charset=utf-8" val DELTA_SHARING_CAPABILITIES_HEADER = "delta-sharing-capabilities" @@ -893,6 +896,10 @@ object DeltaSharingService { } } .toMap + + // TEMPORARY: Log all request headers for debugging X-Client-Region availability + logger.info(s"Request headers: ${JsonUtils.toJson(headerMap)}") + buildClientLocationContext(headerMap, accessLoggingConfig) } From 858eefcea80bc60c7aa61e6f7e94857787490afb Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Tue, 2 Jun 2026 13:52:35 +0300 Subject: [PATCH 07/15] Handle X-Client-Region --- .../sharing/server/DeltaSharingService.scala | 103 +++--- .../sharing/server/config/ServerConfig.scala | 13 +- .../server/telemetry/AccessLogEmitter.scala | 68 ++-- .../server/telemetry/GcpPricingTier.scala | 333 ++++++++++++++++++ .../server/DeltaSharingServiceSuite.scala | 127 +++++-- .../telemetry/AccessLogEmitterSuite.scala | 20 +- .../telemetry/GcpPricingTierSuite.scala | 275 +++++++++++++++ .../conf/delta-sharing-server.yaml.template | 8 + 8 files changed, 824 insertions(+), 123 deletions(-) create mode 100644 server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala create mode 100644 server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 407bbe984..85204afde 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -49,7 +49,7 @@ import io.delta.sharing.server.common.JsonUtils import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ -import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry} +import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry, GcpPricingTier} object ErrorCode { val UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -211,20 +211,16 @@ class DeltaSharingService(serverConfig: ServerConfig) { req, serverConfig.getAccessLogging) - // Emit to access log (new approach) + // Emit to access log val entry = AccessLogEntry( share = share, schema = schema, table = table, egressBytes = bytes, - requestType = AccessLogEmitter.QueryRequestType, timestampMs = nowMs, + pricingTier = location.pricingTier, clientRegion = location.clientRegion, - clientRegionSubdivision = location.clientRegionSubdivision, - clientIp = location.clientIp, - clientIpSource = location.clientIpSource, - clientPricingGroup = location.clientPricingGroup, - clientLocationSource = location.clientLocationSource + requestType = AccessLogEmitter.QueryRequestType ) accessLogEmitter.record(entry) } @@ -245,20 +241,16 @@ class DeltaSharingService(serverConfig: ServerConfig) { req, serverConfig.getAccessLogging) - // Emit to access log immediately (new approach - no aggregation) + // Emit to access log val entry = AccessLogEntry( share = share, schema = schema, table = table, egressBytes = bytes, - requestType = AccessLogEmitter.CdfStreamRequestType, timestampMs = nowMs, + pricingTier = location.pricingTier, clientRegion = location.clientRegion, - clientRegionSubdivision = location.clientRegionSubdivision, - clientIp = location.clientIp, - clientIpSource = location.clientIpSource, - clientPricingGroup = location.clientPricingGroup, - clientLocationSource = location.clientLocationSource + requestType = AccessLogEmitter.CdfStreamRequestType ) accessLogEmitter.record(entry) } @@ -874,13 +866,14 @@ object DeltaSharingService { "CO" -> "latam", "PE" -> "latam") + /** + * Simplified location context for egress pricing. + * @param clientRegion ISO country code (e.g., "US", "MT") from X-Client-Region header + * @param pricingTier GCP egress pricing tier (see GcpPricingTier for values) + */ private[server] case class ClientLocationContext( clientRegion: Option[String], - clientRegionSubdivision: Option[String], - clientIp: Option[String], - clientIpSource: Option[String], - clientPricingGroup: String, - clientLocationSource: Option[String]) + pricingTier: String) private[server] def extractEgressBytes(actions: Seq[Object]): Long = { actions.map(extractActionBytes).sum @@ -897,9 +890,6 @@ object DeltaSharingService { } .toMap - // TEMPORARY: Log all request headers for debugging X-Client-Region availability - logger.info(s"Request headers: ${JsonUtils.toJson(headerMap)}") - buildClientLocationContext(headerMap, accessLoggingConfig) } @@ -911,58 +901,59 @@ object DeltaSharingService { k.toLowerCase(Locale.ROOT) -> v } + // Extract client region from headers val regionHeaders = getOrderedHeaders( cfgOpt.flatMap(c => Option(c.getClientRegionHeader)), DefaultRegionHeaders) - val subdivisionHeaders = getOrderedHeaders( - cfgOpt.flatMap(c => Option(c.getClientRegionSubdivisionHeader)), - DefaultSubdivisionHeaders) val ipHeaders = getOrderedHeaders( cfgOpt.flatMap(c => Option(c.getClientIpHeader)), DefaultIpHeaders) val regionWithSource = getFirstLocation(regionHeaders, normalizedHeaders) - val subdivisionWithSource = getFirstLocation(subdivisionHeaders, normalizedHeaders) val ipWithSource = getFirstClientIp(ipHeaders, normalizedHeaders) val region = regionWithSource.map(_._1) - val subdivision = subdivisionWithSource.map(_._1) val clientIp = ipWithSource.map(_._1) - val clientIpSource = ipWithSource.map(_._2) - val source = regionWithSource.map(_._2).orElse(subdivisionWithSource.map(_._2)) - - val configuredPricingMap = cfgOpt - .flatMap(c => Option(c.getPricingGroups)) - .map(_.asScala.toMap) - .getOrElse(Map.empty[String, String]) - .map { case (k, v) => normalizeLocationCode(k) -> v.trim } - .filter { case (k, v) => k.nonEmpty && v.nonEmpty } - - val pricingMap = if (configuredPricingMap.nonEmpty) { - configuredPricingMap - } else { - DefaultPricingGroupsByRegion - } - val defaultPricingGroup = cfgOpt - .flatMap(c => Option(c.getDefaultPricingGroup)) + // GCP Pricing Tier calculation + val sourceRegion = cfgOpt + .flatMap(c => Option(c.getSourceRegion)) .map(_.trim) .filter(_.nonEmpty) - .getOrElse("unknown") - val pricingGroup = subdivision.flatMap(pricingMap.get) - .orElse(region.flatMap(pricingMap.get)) - .orElse(pricingMap.get("*")) - .orElse(region) - .getOrElse(defaultPricingGroup) + val detectGcpTraffic = cfgOpt + .map(c => c.getDetectGcpTraffic) + .getOrElse(true) + + val envoyPeerMetadata = normalizedHeaders.get("x-envoy-peer-metadata") + + val (egressType, destGcpRegion) = GcpPricingTier.determineEgressType( + clientIp, + envoyPeerMetadata, + region, + detectGcpTraffic) + + // Calculate pricing tier: + // - For internet traffic: based on destination only (no sourceRegion needed) + // - For inter-region traffic: requires sourceRegion + // - For same_zone/same_region: tier is the egress type itself + val pricingTier = egressType match { + case GcpPricingTier.EgressType.INTERNET => + GcpPricingTier.calculatePricingTier( + sourceRegion, destGcpRegion, region, egressType) + case GcpPricingTier.EgressType.SAME_ZONE | + GcpPricingTier.EgressType.SAME_REGION => + egressType.toString + case GcpPricingTier.EgressType.INTER_REGION if sourceRegion.isDefined => + GcpPricingTier.calculatePricingTier( + sourceRegion, destGcpRegion, region, egressType) + case _ => + "unknown" + } ClientLocationContext( clientRegion = region, - clientRegionSubdivision = subdivision, - clientIp = clientIp, - clientIpSource = clientIpSource, - clientPricingGroup = pricingGroup, - clientLocationSource = source) + pricingTier = pricingTier) } private def getOrderedHeaders( diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index 5acd545b2..4802f6706 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -148,7 +148,14 @@ case class AccessLoggingConfig( // A wildcard key "*" can be used as a catch-all default. @BeanProperty var pricingGroups: java.util.Map[String, String], // Fallback group when no mapping and no region are available. - @BeanProperty var defaultPricingGroup: String) extends ConfigItem { + @BeanProperty var defaultPricingGroup: String, + // The GCP region where this server runs (for example: us-central1). + // Used for pricing tier calculation based on source→destination pairs. + @BeanProperty var sourceRegion: String, + // Enable GCP inter-region traffic detection via X-Envoy-Peer-Metadata header. + // When true, traffic from other GCP regions is classified as inter-region (cheaper) + // rather than internet egress. + @BeanProperty var detectGcpTraffic: Boolean) extends ConfigItem { def this() = { this( @@ -157,7 +164,9 @@ case class AccessLoggingConfig( clientRegionSubdivisionHeader = "x-client-region-subdivision", clientIpHeader = "x-forwarded-for", pricingGroups = Collections.emptyMap(), - defaultPricingGroup = "unknown") + defaultPricingGroup = "unknown", + sourceRegion = "", + detectGcpTraffic = true) } override def checkConfig(): Unit = { diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index 791649aa0..fb0dd4b74 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -24,32 +24,52 @@ import io.delta.sharing.server.config.ServerConfig /** * Represents a single access log entry for share data egress tracking. * + * == Core Fields (always present) == * @param share The name of the share being accessed * @param schema The schema containing the table * @param table The table being accessed - * @param egressBytes The number of bytes transferred - * @param requestType The type of request: "query" or "cdf_stream" + * @param egressBytes The number of bytes transferred (used for cost calculation) * @param timestampMs The timestamp of the access in milliseconds since epoch - * @param clientRegion Optional client region code derived from request headers (for example: US) - * @param clientRegionSubdivision Optional client subdivision code (for example: USCA) - * @param clientIp Optional client IP derived from forwarding headers - * @param clientIpSource Optional header name used to resolve client IP - * @param clientPricingGroup Pricing group label used for downstream egress cost attribution - * @param clientLocationSource Optional header name used to resolve client location + * + * == Pricing Fields == + * @param pricingTier GCP egress pricing tier. See GcpPricingTier for values: + * + * '''Internet Egress (Premium Tier):''' + * - `internet_to_na_eu` - To North America or Europe (~$0.12/GiB) + * - `internet_to_apac` - To Asia Pacific (~$0.12/GiB) + * - `internet_to_latam` - To Latin America (~$0.19/GiB) + * - `internet_to_oceania` - To Australia/Oceania (~$0.15/GiB) + * + * '''Inter-region GCP Traffic:''' + * - `interregion_na_to_na` - NA to NA (~$0.02/GiB) + * - `interregion_na_to_eu` - NA to EU (~$0.05/GiB) + * - `interregion_eu_to_na` - EU to NA (~$0.05/GiB) + * - `interregion_eu_to_eu` - EU to EU (~$0.02/GiB) + * - `interregion_to_apac` - Any to APAC (~$0.08/GiB) + * - `interregion_to_oceania` - Any to Oceania (~$0.10/GiB) + * - `interregion_to_latam` - Any to LATAM (~$0.14/GiB) + * + * '''Free/Low-cost:''' + * - `same_zone` - Within same GCP zone (free) + * - `same_region` - Within same region or K8s cluster (~free) + * + * - `unknown` - Could not determine pricing tier + * + * Pricing reference: https://cloud.google.com/vpc/network-pricing + * + * == Optional Context == + * @param clientRegion ISO 3166-1 alpha-2 country code of the client (e.g., "US", "MT") + * @param requestType Type of request: "query" (snapshot read) or "cdf_stream" (CDF streaming) */ case class AccessLogEntry( share: String, schema: String, table: String, egressBytes: Long, - requestType: String, - timestampMs: Long, - clientRegion: Option[String] = None, - clientRegionSubdivision: Option[String] = None, - clientIp: Option[String] = None, - clientIpSource: Option[String] = None, - clientPricingGroup: String = "unknown", - clientLocationSource: Option[String] = None) + timestampMs: Long, + pricingTier: String = "unknown", + clientRegion: Option[String] = None, + requestType: String = "query") /** * Trait for emitting access logs for share data egress tracking. @@ -96,26 +116,24 @@ class JsonAccessLogEmitter extends AccessLogEmitter { return } + // Core payload with essential fields for cost tracking val basePayload = Map( "logType" -> "ACCESS_LOG", "share" -> entry.share, "schema" -> entry.schema, "table" -> entry.table, "egressBytes" -> entry.egressBytes, - "requestType" -> entry.requestType, + "pricingTier" -> entry.pricingTier, "timestampMs" -> entry.timestampMs, - "clientPricingGroup" -> entry.clientPricingGroup + "requestType" -> entry.requestType ) - val locationPayload = Seq( - entry.clientRegion.map("clientRegion" -> _), - entry.clientRegionSubdivision.map("clientRegionSubdivision" -> _), - entry.clientIp.map("clientIp" -> _), - entry.clientIpSource.map("clientIpSource" -> _), - entry.clientLocationSource.map("clientLocationSource" -> _) + // Optional context fields + val contextPayload = Seq( + entry.clientRegion.map("clientRegion" -> _) ).flatten.toMap - val logPayload = basePayload ++ locationPayload + val logPayload = basePayload ++ contextPayload logger.info(JsonUtils.toJson(logPayload)) } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala new file mode 100644 index 000000000..7dee2cbd9 --- /dev/null +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -0,0 +1,333 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import java.util.{Base64, Locale} + +import scala.util.Try + +/** + * GCP pricing tier calculation for egress traffic. + * + * Implements classification of network egress into GCP pricing tiers based on: + * - Source region (where the data originates, e.g., GCS bucket location) + * - Destination region (where the client is located) + * - Traffic type (same-zone, inter-region GCP, or internet) + * + * Pricing tiers are based on: + * https://cloud.google.com/vpc/pricing-announce + */ +object GcpPricingTier { + + /** + * Continent groupings for GCP regions and country codes. + */ + object Continent extends Enumeration { + type Continent = Value + val NA, EU, APAC, LATAM, OCEANIA = Value + val UNKNOWN_CONTINENT: Continent = Value("UNKNOWN") + } + + import Continent._ + + /** + * Traffic types for egress pricing. + */ + object EgressType extends Enumeration { + type EgressType = Value + val SAME_ZONE = Value("same_zone") + val SAME_REGION = Value("same_region") + val INTER_REGION = Value("inter_region") + val INTERNET = Value("internet") + val UNKNOWN_TYPE: EgressType = Value("unknown") + } + + import EgressType._ + + /** + * Map GCP region prefix to continent. + * GCP region format: {location}-{zone_letter}{zone_number} (e.g., us-central1-a) + * We extract the location prefix. + */ + private val GcpRegionToContinent: Map[String, Continent] = Map( + // North America + "us" -> NA, + "northamerica" -> NA, + // Europe + "europe" -> EU, + // Asia Pacific + "asia" -> APAC, + // South America + "southamerica" -> LATAM, + // Australia / Oceania + "australia" -> OCEANIA + ) + + /** + * Map ISO 3166-1 alpha-2 country codes to continents. + * This covers the major countries; for others, we default to UNKNOWN. + */ + private val CountryToContinent: Map[String, Continent] = Map( + // North America + "US" -> NA, "CA" -> NA, "MX" -> NA, + // Europe + "GB" -> EU, "IE" -> EU, "DE" -> EU, "FR" -> EU, "NL" -> EU, "BE" -> EU, + "CH" -> EU, "AT" -> EU, "ES" -> EU, "PT" -> EU, "IT" -> EU, "SE" -> EU, + "NO" -> EU, "DK" -> EU, "FI" -> EU, "PL" -> EU, "CZ" -> EU, "HU" -> EU, + "RO" -> EU, "BG" -> EU, "GR" -> EU, "HR" -> EU, "SK" -> EU, "SI" -> EU, + "EE" -> EU, "LV" -> EU, "LT" -> EU, "LU" -> EU, "MT" -> EU, "CY" -> EU, + "IS" -> EU, "UA" -> EU, "BY" -> EU, "RU" -> EU, + // Asia Pacific + "JP" -> APAC, "KR" -> APAC, "CN" -> APAC, "HK" -> APAC, "TW" -> APAC, + "SG" -> APAC, "MY" -> APAC, "ID" -> APAC, "TH" -> APAC, "VN" -> APAC, + "PH" -> APAC, "IN" -> APAC, "PK" -> APAC, "BD" -> APAC, + // Oceania + "AU" -> OCEANIA, "NZ" -> OCEANIA, + // Latin America + "BR" -> LATAM, "AR" -> LATAM, "CL" -> LATAM, "CO" -> LATAM, "PE" -> LATAM, + "VE" -> LATAM, "EC" -> LATAM, "BO" -> LATAM, "PY" -> LATAM, "UY" -> LATAM, + "CR" -> LATAM, "PA" -> LATAM, "GT" -> LATAM, "HN" -> LATAM, "SV" -> LATAM, + "NI" -> LATAM, "DO" -> LATAM, "CU" -> LATAM, "PR" -> LATAM + ) + + /** + * Pricing tiers matching GCP network egress pricing structure. + * + * Inter-region format: interregion_{source}_{dest} or interregion_to_{dest} + * Internet format: internet_to_{dest} (pricing based on destination only) + * Special cases: same_zone, same_region (both essentially free) + */ + val TIER_SAME_ZONE = "same_zone" + val TIER_SAME_REGION = "same_region" + val TIER_INTERREGION_NA_NA = "interregion_na_to_na" + val TIER_INTERREGION_NA_EU = "interregion_na_to_eu" + val TIER_INTERREGION_EU_NA = "interregion_eu_to_na" + val TIER_INTERREGION_EU_EU = "interregion_eu_to_eu" + val TIER_INTERREGION_APAC = "interregion_to_apac" + val TIER_INTERREGION_TO_OCEANIA = "interregion_to_oceania" + val TIER_INTERREGION_TO_LATAM = "interregion_to_latam" + val TIER_INTERNET_NA_EU = "internet_to_na_eu" + val TIER_INTERNET_APAC = "internet_to_apac" + val TIER_INTERNET_LATAM = "internet_to_latam" + val TIER_INTERNET_OCEANIA = "internet_to_oceania" + val TIER_UNKNOWN = "unknown" + + /** + * Extract continent from a GCP region string (e.g., "us-central1", "europe-west1-b"). + */ + def continentFromGcpRegion(region: String): Continent = { + if (region == null || region.isEmpty) return UNKNOWN_CONTINENT + + val normalized = region.toLowerCase(Locale.ROOT) + // GCP region format: {continent_prefix}-{location}{number} + // e.g., us-central1, europe-west1, asia-east1, australia-southeast1 + val prefix = normalized.split("-").headOption.getOrElse("") + + GcpRegionToContinent.getOrElse(prefix, UNKNOWN_CONTINENT) + } + + /** + * Extract continent from an ISO 3166-1 alpha-2 country code. + */ + def continentFromCountryCode(countryCode: String): Continent = { + if (countryCode == null || countryCode.isEmpty) return UNKNOWN_CONTINENT + + val normalized = countryCode.toUpperCase(Locale.ROOT).trim + // Handle special "ZZ" code (unknown location from GCP load balancer) + if (normalized == "ZZ") return UNKNOWN_CONTINENT + + CountryToContinent.getOrElse(normalized, UNKNOWN_CONTINENT) + } + + /** + * Calculate the pricing tier based on source region, destination, and traffic type. + * + * @param sourceRegion The GCP region where data originates (e.g., "us-central1") + * @param destinationRegion Optional GCP region for inter-region GCP traffic + * @param destinationCountry Optional ISO country code for internet traffic + * @param egressType The type of egress (same_zone, inter_region, internet) + * @return The pricing tier string + */ + def calculatePricingTier( + sourceRegion: Option[String], + destinationRegion: Option[String], + destinationCountry: Option[String], + egressType: EgressType): String = { + + egressType match { + case SAME_ZONE => TIER_SAME_ZONE + case SAME_REGION => TIER_SAME_REGION + + case INTER_REGION => + val srcContinent = sourceRegion + .map(continentFromGcpRegion).getOrElse(UNKNOWN_CONTINENT) + val dstContinent = destinationRegion + .map(continentFromGcpRegion).getOrElse(UNKNOWN_CONTINENT) + calculateInterRegionTier(srcContinent, dstContinent) + + case INTERNET => + // Internet egress pricing depends only on destination, not source + val dstContinent = destinationCountry + .map(continentFromCountryCode).getOrElse(UNKNOWN_CONTINENT) + calculateInternetTier(dstContinent) + + case UNKNOWN_TYPE => TIER_UNKNOWN + } + } + + /** + * Calculate inter-region GCP pricing tier based on source and destination continents. + * + * GCP inter-region pricing (approximate): + * - NA to NA: $0.02/GiB + * - NA to EU: $0.05/GiB + * - EU to EU: $0.02/GiB + * - To Asia: $0.08/GiB + * - To Australia: $0.10/GiB + * - To LATAM: $0.14/GiB + */ + private def calculateInterRegionTier(src: Continent, dst: Continent): String = { + (src, dst) match { + case (NA, NA) => TIER_INTERREGION_NA_NA + case (NA, EU) => TIER_INTERREGION_NA_EU + case (EU, NA) => TIER_INTERREGION_EU_NA + case (EU, EU) => TIER_INTERREGION_EU_EU + case (APAC, APAC) => TIER_INTERREGION_APAC + case (_, OCEANIA) => TIER_INTERREGION_TO_OCEANIA + case (_, LATAM) => TIER_INTERREGION_TO_LATAM + case (_, APAC) => TIER_INTERREGION_APAC + case _ => TIER_UNKNOWN + } + } + + /** + * Calculate internet egress pricing tier based on destination continent. + * Note: GCP internet egress pricing depends only on destination, not source. + * + * GCP Premium Tier Internet Egress (approximate): + * - To NA/EU: $0.12/GiB + * - To Asia: $0.12/GiB + * - To LATAM: $0.19/GiB + * - To Australia: $0.15/GiB + */ + def calculateInternetTier(dst: Continent): String = { + dst match { + case NA | EU => TIER_INTERNET_NA_EU + case APAC => TIER_INTERNET_APAC + case LATAM => TIER_INTERNET_LATAM + case OCEANIA => TIER_INTERNET_OCEANIA + case UNKNOWN_CONTINENT => TIER_UNKNOWN + } + } + + /** + * Parse X-Envoy-Peer-Metadata header to extract GCP region. + * + * The header is base64-encoded and contains structured metadata including + * a "gcp_location" field with the region (e.g., "us-central1-f"). + * + * @param headerValue The base64-encoded X-Envoy-Peer-Metadata header value + * @return Optional GCP region extracted from the metadata + */ + def extractGcpRegionFromEnvoyMetadata(headerValue: String): Option[String] = { + if (headerValue == null || headerValue.isEmpty) return None + + Try { + val decoded = new String(Base64.getDecoder.decode(headerValue), "UTF-8") + // The metadata is typically in a protobuf-like format or JSON + // Look for patterns like "gcp_location":"us-central1-f" or gcp_location:us-central1-f + val gcpLocationPattern = """(?:gcp_location|GCP_LOCATION)["\s:]+([a-z]+-[a-z0-9-]+)""".r + gcpLocationPattern.findFirstMatchIn(decoded).map { m => + // Extract region without zone suffix (e.g., "us-central1" from "us-central1-f") + val fullLocation = m.group(1) + // Remove zone letter if present (e.g., -a, -b, -f) + fullLocation.replaceAll("-[a-z]$", "") + } + }.toOption.flatten + } + + /** + * Check if an IP address appears to be from the same Kubernetes cluster. + * + * This is a heuristic check based on common pod CIDR ranges. + * In practice, same-cluster traffic often has no X-Forwarded-For header + * or has a private IP in the forwarding chain. + * + * @param clientIp The client IP address + * @return true if the IP appears to be from the same cluster + */ + def isLikelySameClusterTraffic(clientIp: Option[String]): Boolean = { + clientIp match { + case None => true // No forwarding header often means same-cluster + case Some(ip) => + // Check for common private IP ranges used in Kubernetes + ip.startsWith("10.") || + ip.startsWith("172.16.") || ip.startsWith("172.17.") || ip.startsWith("172.18.") || + ip.startsWith("172.19.") || ip.startsWith("172.20.") || ip.startsWith("172.21.") || + ip.startsWith("172.22.") || ip.startsWith("172.23.") || ip.startsWith("172.24.") || + ip.startsWith("172.25.") || ip.startsWith("172.26.") || ip.startsWith("172.27.") || + ip.startsWith("172.28.") || ip.startsWith("172.29.") || ip.startsWith("172.30.") || + ip.startsWith("172.31.") || + ip.startsWith("192.168.") || + ip == "127.0.0.1" || ip == "::1" + } + } + + /** + * Determine the egress type based on available information. + * + * @param clientIp The client IP address (from X-Forwarded-For) + * @param envoyPeerMetadata The X-Envoy-Peer-Metadata header value + * @param clientRegion The client region from X-Client-Region header + * @param detectGcpTraffic Whether GCP traffic detection is enabled + * @return Tuple of (EgressType, Optional destination GCP region) + */ + def determineEgressType( + clientIp: Option[String], + envoyPeerMetadata: Option[String], + clientRegion: Option[String], + detectGcpTraffic: Boolean): (EgressType, Option[String]) = { + + // Check for same-cluster traffic first + if (isLikelySameClusterTraffic(clientIp)) { + return (SAME_REGION, None) + } + + // If GCP traffic detection is enabled, try to extract region from Envoy metadata + if (detectGcpTraffic) { + val gcpRegion = envoyPeerMetadata.flatMap(extractGcpRegionFromEnvoyMetadata) + if (gcpRegion.isDefined) { + return (INTER_REGION, gcpRegion) + } + } + + // Check if X-Client-Region indicates unknown/internal (ZZ) + clientRegion match { + // scalastyle:off caselocale + case Some(region) if region.toUpperCase == "ZZ" => + // scalastyle:on caselocale + // ZZ typically means GCP couldn't determine the location (internal traffic) + (INTER_REGION, None) + case Some(_) => + // Has a valid country code - this is internet traffic + (INTERNET, None) + case None => + // No region header - could be internal or misconfigured + (UNKNOWN_TYPE, None) + } + } +} diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index a2e7f550b..fb46330aa 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -274,53 +274,45 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { test("buildClientLocationContext uses configured geo headers and pricing map") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) - cfg.setPricingGroups(Map("US" -> "na-tier-1", "USCA" -> "us-west").asJava) val ctx = DeltaSharingService.buildClientLocationContext( Map( "X-Client-Region" -> "us", - "X-Client-Region-Subdivision" -> "us-ca" + "X-Forwarded-For" -> "1.2.3.4" // Public IP -> internet traffic ), cfg) assert(ctx.clientRegion.contains("US")) - assert(ctx.clientRegionSubdivision.contains("USCA")) - assert(ctx.clientIp.isEmpty) - assert(ctx.clientPricingGroup == "us-west") - assert(ctx.clientLocationSource.contains("x-client-region")) + assert(ctx.pricingTier == "internet_to_na_eu") } - test("buildClientLocationContext falls back to default headers and wildcard pricing") { + test("buildClientLocationContext falls back to default headers for region") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) - cfg.setPricingGroups(Map("*" -> "global-tier-2").asJava) val ctx = DeltaSharingService.buildClientLocationContext( - Map("CF-IPCountry" -> "br"), + Map( + "CF-IPCountry" -> "br", + "X-Forwarded-For" -> "1.2.3.4" // Public IP -> internet traffic + ), cfg) assert(ctx.clientRegion.contains("BR")) - assert(ctx.clientRegionSubdivision.isEmpty) - assert(ctx.clientIp.isEmpty) - assert(ctx.clientPricingGroup == "global-tier-2") - assert(ctx.clientLocationSource.contains("cf-ipcountry")) + assert(ctx.pricingTier == "internet_to_latam") } - test("buildClientLocationContext uses fallback pricing group when location is unavailable") { + test("buildClientLocationContext returns same_region for local traffic") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) - cfg.setDefaultPricingGroup("unknown-tier") + // No region header, no X-Forwarded-For -> same_region val ctx = DeltaSharingService.buildClientLocationContext(Map.empty[String, String], cfg) assert(ctx.clientRegion.isEmpty) - assert(ctx.clientRegionSubdivision.isEmpty) - assert(ctx.clientIp.isEmpty) - assert(ctx.clientPricingGroup == "unknown-tier") - assert(ctx.clientLocationSource.isEmpty) + assert(ctx.pricingTier == "same_region") } - test("buildClientLocationContext extracts client IP from forwarding headers") { + test("buildClientLocationContext uses IP from X-Forwarded-For for pricing tier") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) @@ -329,20 +321,105 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { cfg) assert(ctx.clientRegion.isEmpty) - assert(ctx.clientIp.contains("217.9.4.27")) - assert(ctx.clientIpSource.contains("x-forwarded-for")) + // Public IP in chain -> internet traffic, but no region -> unknown destination + assert(ctx.pricingTier == "unknown") } - test("buildClientLocationContext applies default pricing groups when no custom map") { + test("buildClientLocationContext calculates internet tier for EU destination") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) val ctx = DeltaSharingService.buildClientLocationContext( - Map("X-Appengine-Country" -> "de"), + Map( + "X-Appengine-Country" -> "de", + "X-Forwarded-For" -> "1.2.3.4" // Public IP -> internet traffic + ), cfg) assert(ctx.clientRegion.contains("DE")) - assert(ctx.clientPricingGroup == "na_eu") + assert(ctx.pricingTier == "internet_to_na_eu") + } + + test("buildClientLocationContext calculates internet tier for DE destination") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setSourceRegion("us-central1") + cfg.setDetectGcpTraffic(true) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map( + "X-Client-Region" -> "DE", + "X-Forwarded-For" -> "1.2.3.4" + ), + cfg) + + assert(ctx.clientRegion.contains("DE")) + assert(ctx.pricingTier == "internet_to_na_eu") + } + + test("buildClientLocationContext detects same_region for no external IP") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setSourceRegion("us-central1") + cfg.setDetectGcpTraffic(true) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map.empty[String, String], + cfg) + + assert(ctx.pricingTier == "same_region") + } + + test("buildClientLocationContext calculates inter-region tier for ZZ region") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setSourceRegion("us-central1") + cfg.setDetectGcpTraffic(true) + + val ctx = DeltaSharingService.buildClientLocationContext( + Map( + "X-Client-Region" -> "ZZ", + "X-Forwarded-For" -> "34.100.0.1" + ), + cfg) + + assert(ctx.clientRegion.contains("ZZ")) + // Inter-region from NA (us-central1) to unknown -> internet fallback + assert(ctx.pricingTier.startsWith("inter") || ctx.pricingTier == "unknown") + } + + test("buildClientLocationContext calculates internet tier without source region") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setDetectGcpTraffic(true) + // Note: sourceRegion is NOT set + + val ctx = DeltaSharingService.buildClientLocationContext( + Map( + "X-Client-Region" -> "DE", + "X-Forwarded-For" -> "1.2.3.4" + ), + cfg) + + // Internet pricing doesn't require sourceRegion + assert(ctx.pricingTier == "internet_to_na_eu") + } + + test("buildClientLocationContext returns unknown for inter-region without source") { + val cfg = new AccessLoggingConfig() + cfg.setEnabled(true) + cfg.setDetectGcpTraffic(true) + // Note: sourceRegion is NOT set + + val ctx = DeltaSharingService.buildClientLocationContext( + Map( + "X-Client-Region" -> "ZZ", // ZZ indicates internal GCP traffic + "X-Forwarded-For" -> "34.100.0.1" + ), + cfg) + + // Inter-region requires sourceRegion, so unknown pricing tier + assert(ctx.pricingTier == "unknown") } integrationTest("401 Unauthorized Error: incorrect token") { diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala index f4c28ecf1..da2762746 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/AccessLogEmitterSuite.scala @@ -28,14 +28,10 @@ class AccessLogEmitterSuite extends FunSuite { schema = "test-schema", table = "test-table", egressBytes = 1024L, - requestType = AccessLogEmitter.QueryRequestType, timestampMs = System.currentTimeMillis(), + pricingTier = "internet_to_na_eu", clientRegion = Some("US"), - clientRegionSubdivision = Some("USCA"), - clientIp = Some("217.9.4.27"), - clientIpSource = Some("x-forwarded-for"), - clientPricingGroup = "na-tier-1", - clientLocationSource = Some("x-client-region") + requestType = AccessLogEmitter.QueryRequestType ) assert(entry.share == "test-share") @@ -44,11 +40,7 @@ class AccessLogEmitterSuite extends FunSuite { assert(entry.egressBytes == 1024L) assert(entry.requestType == "query") assert(entry.clientRegion.contains("US")) - assert(entry.clientRegionSubdivision.contains("USCA")) - assert(entry.clientIp.contains("217.9.4.27")) - assert(entry.clientIpSource.contains("x-forwarded-for")) - assert(entry.clientPricingGroup == "na-tier-1") - assert(entry.clientLocationSource.contains("x-client-region")) + assert(entry.pricingTier == "internet_to_na_eu") } test("NoopAccessLogEmitter ignores all records") { @@ -58,7 +50,6 @@ class AccessLogEmitterSuite extends FunSuite { schema = "sc", table = "t", egressBytes = 100L, - requestType = AccessLogEmitter.QueryRequestType, timestampMs = 0L )) } @@ -71,7 +62,6 @@ class AccessLogEmitterSuite extends FunSuite { schema = "sc", table = "t", egressBytes = 0L, - requestType = AccessLogEmitter.QueryRequestType, timestampMs = 0L )) } @@ -84,8 +74,8 @@ class AccessLogEmitterSuite extends FunSuite { schema = "analytics", table = "events", egressBytes = 50000L, - requestType = AccessLogEmitter.CdfStreamRequestType, - timestampMs = 1716864000000L + timestampMs = 1716864000000L, + requestType = AccessLogEmitter.CdfStreamRequestType )) } diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala new file mode 100644 index 000000000..f99eba92b --- /dev/null +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala @@ -0,0 +1,275 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import java.util.Base64 + +import org.scalatest.FunSuite + +class GcpPricingTierSuite extends FunSuite { + import GcpPricingTier._ + import GcpPricingTier.Continent._ + import GcpPricingTier.EgressType._ + + // ===== Continent Mapping Tests ===== + + test("continentFromGcpRegion maps US regions to NA") { + assert(continentFromGcpRegion("us-central1") == NA) + assert(continentFromGcpRegion("us-east1") == NA) + assert(continentFromGcpRegion("us-west1-a") == NA) + } + + test("continentFromGcpRegion maps Europe regions to EU") { + assert(continentFromGcpRegion("europe-west1") == EU) + assert(continentFromGcpRegion("europe-north1-b") == EU) + } + + test("continentFromGcpRegion maps Asia regions to APAC") { + assert(continentFromGcpRegion("asia-east1") == APAC) + assert(continentFromGcpRegion("asia-southeast1") == APAC) + } + + test("continentFromGcpRegion maps Australia regions to OCEANIA") { + assert(continentFromGcpRegion("australia-southeast1") == OCEANIA) + } + + test("continentFromGcpRegion maps South America regions to LATAM") { + assert(continentFromGcpRegion("southamerica-east1") == LATAM) + } + + test("continentFromGcpRegion handles null and empty") { + assert(continentFromGcpRegion(null) == UNKNOWN_CONTINENT) + assert(continentFromGcpRegion("") == UNKNOWN_CONTINENT) + assert(continentFromGcpRegion("unknown-region") == UNKNOWN_CONTINENT) + } + + test("continentFromCountryCode maps NA countries correctly") { + assert(continentFromCountryCode("US") == NA) + assert(continentFromCountryCode("CA") == NA) + assert(continentFromCountryCode("MX") == NA) + } + + test("continentFromCountryCode maps EU countries correctly") { + assert(continentFromCountryCode("DE") == EU) + assert(continentFromCountryCode("FR") == EU) + assert(continentFromCountryCode("GB") == EU) + assert(continentFromCountryCode("MT") == EU) + } + + test("continentFromCountryCode maps APAC countries correctly") { + assert(continentFromCountryCode("JP") == APAC) + assert(continentFromCountryCode("SG") == APAC) + assert(continentFromCountryCode("IN") == APAC) + } + + test("continentFromCountryCode maps Oceania countries correctly") { + assert(continentFromCountryCode("AU") == OCEANIA) + assert(continentFromCountryCode("NZ") == OCEANIA) + } + + test("continentFromCountryCode maps LATAM countries correctly") { + assert(continentFromCountryCode("BR") == LATAM) + assert(continentFromCountryCode("AR") == LATAM) + } + + test("continentFromCountryCode handles ZZ (unknown) code") { + assert(continentFromCountryCode("ZZ") == UNKNOWN_CONTINENT) + } + + test("continentFromCountryCode is case-insensitive") { + assert(continentFromCountryCode("us") == NA) + assert(continentFromCountryCode("Us") == NA) + } + + // ===== Pricing Tier Calculation Tests ===== + + test("calculatePricingTier returns same_zone for SAME_ZONE egress type") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = None, + egressType = SAME_ZONE) + assert(tier == TIER_SAME_ZONE) + } + + test("calculatePricingTier returns same_region for SAME_REGION egress type") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = None, + egressType = SAME_REGION) + assert(tier == TIER_SAME_REGION) + } + + test("calculatePricingTier returns interregion_na_na for NA to NA inter-region") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = Some("us-east1"), + destinationCountry = None, + egressType = INTER_REGION) + assert(tier == TIER_INTERREGION_NA_NA) + } + + test("calculatePricingTier returns interregion_na_eu for NA to EU inter-region") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = Some("europe-west1"), + destinationCountry = None, + egressType = INTER_REGION) + assert(tier == TIER_INTERREGION_NA_EU) + } + + test("calculatePricingTier returns interregion_eu_na for EU to NA inter-region") { + val tier = calculatePricingTier( + sourceRegion = Some("europe-west1"), + destinationRegion = Some("us-east1"), + destinationCountry = None, + egressType = INTER_REGION) + assert(tier == TIER_INTERREGION_EU_NA) + } + + test("calculatePricingTier returns interregion_apac for Asia to Asia inter-region") { + val tier = calculatePricingTier( + sourceRegion = Some("asia-east1"), + destinationRegion = Some("asia-southeast1"), + destinationCountry = None, + egressType = INTER_REGION) + assert(tier == TIER_INTERREGION_APAC) + } + + test("calculatePricingTier returns interregion_to_oceania for any to Oceania inter-region") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = Some("australia-southeast1"), + destinationCountry = None, + egressType = INTER_REGION) + assert(tier == TIER_INTERREGION_TO_OCEANIA) + } + + test("calculatePricingTier returns internet_to_na_eu for internet egress to NA/EU") { + val tierNA = calculatePricingTier( + sourceRegion = Some("europe-west1"), + destinationRegion = None, + destinationCountry = Some("US"), + egressType = INTERNET) + assert(tierNA == TIER_INTERNET_NA_EU) + + val tierEU = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = Some("DE"), + egressType = INTERNET) + assert(tierEU == TIER_INTERNET_NA_EU) + } + + test("calculatePricingTier returns internet_apac for internet egress to Asia") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = Some("JP"), + egressType = INTERNET) + assert(tier == TIER_INTERNET_APAC) + } + + test("calculatePricingTier returns internet_latam for internet egress to LATAM") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = Some("BR"), + egressType = INTERNET) + assert(tier == TIER_INTERNET_LATAM) + } + + test("calculatePricingTier returns internet_oceania for internet egress to Oceania") { + val tier = calculatePricingTier( + sourceRegion = Some("us-central1"), + destinationRegion = None, + destinationCountry = Some("AU"), + egressType = INTERNET) + assert(tier == TIER_INTERNET_OCEANIA) + } + + // ===== Egress Type Detection Tests ===== + + test("isLikelySameClusterTraffic returns true for missing client IP") { + assert(isLikelySameClusterTraffic(None)) + } + + test("isLikelySameClusterTraffic returns true for private IPs") { + assert(isLikelySameClusterTraffic(Some("10.0.0.1"))) + assert(isLikelySameClusterTraffic(Some("172.16.0.1"))) + assert(isLikelySameClusterTraffic(Some("192.168.1.1"))) + assert(isLikelySameClusterTraffic(Some("127.0.0.1"))) + } + + test("isLikelySameClusterTraffic returns false for public IPs") { + assert(!isLikelySameClusterTraffic(Some("8.8.8.8"))) + assert(!isLikelySameClusterTraffic(Some("1.2.3.4"))) + } + + test("determineEgressType returns SAME_REGION for same-cluster traffic") { + val (egressType, destRegion) = determineEgressType( + clientIp = None, + envoyPeerMetadata = None, + clientRegion = None, + detectGcpTraffic = true) + assert(egressType == SAME_REGION) + assert(destRegion.isEmpty) + } + + test("determineEgressType returns INTER_REGION when ZZ region detected") { + val (egressType, destRegion) = determineEgressType( + clientIp = Some("34.100.0.1"), + envoyPeerMetadata = None, + clientRegion = Some("ZZ"), + detectGcpTraffic = true) + assert(egressType == INTER_REGION) + assert(destRegion.isEmpty) + } + + test("determineEgressType returns INTERNET when valid country code present") { + val (egressType, destRegion) = determineEgressType( + clientIp = Some("1.2.3.4"), + envoyPeerMetadata = None, + clientRegion = Some("US"), + detectGcpTraffic = true) + assert(egressType == INTERNET) + assert(destRegion.isEmpty) + } + + // ===== Envoy Metadata Parsing Tests ===== + + test("extractGcpRegionFromEnvoyMetadata returns None for null/empty") { + assert(extractGcpRegionFromEnvoyMetadata(null).isEmpty) + assert(extractGcpRegionFromEnvoyMetadata("").isEmpty) + } + + test("extractGcpRegionFromEnvoyMetadata extracts region from base64 metadata") { + // Create a simple test payload with gcp_location + val payload = """{"gcp_location":"us-central1-f"}""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + val result = extractGcpRegionFromEnvoyMetadata(encoded) + assert(result.contains("us-central1")) + } + + test("extractGcpRegionFromEnvoyMetadata strips zone suffix") { + val payload = """gcp_location:europe-west1-b""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + val result = extractGcpRegionFromEnvoyMetadata(encoded) + assert(result.contains("europe-west1")) + } +} diff --git a/server/src/universal/conf/delta-sharing-server.yaml.template b/server/src/universal/conf/delta-sharing-server.yaml.template index 73d774aa3..40b0b565b 100644 --- a/server/src/universal/conf/delta-sharing-server.yaml.template +++ b/server/src/universal/conf/delta-sharing-server.yaml.template @@ -86,3 +86,11 @@ accessLogging: pricingGroups: {} # Fallback pricing group when no location can be resolved. defaultPricingGroup: "unknown" + # GCP region where this server runs (for example: us-central1). + # Required for GCP pricing tier calculation. + # Used to determine source→destination pairs for egress cost attribution. + sourceRegion: "" + # Enable GCP inter-region traffic detection via X-Envoy-Peer-Metadata header. + # When true, traffic from other GCP regions is classified as inter-region (cheaper) + # rather than internet egress. Set to false to disable detection. + detectGcpTraffic: true From 7d480101e12fd4de4c65f3a97f5730598abadb4b Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Wed, 3 Jun 2026 12:03:18 +0300 Subject: [PATCH 08/15] Add context debug log --- manifests/base/configmap.yaml | 5 +- manifests/zcloud-emea/configmap.yaml | 29 ++++ manifests/zcloud-emea/kustomization.yaml | 2 + manifests/zcloud-prod/configmap.yaml | 5 +- manifests/zcloud-prod2/configmap.yaml | 5 +- manifests/zcloud-prod3/configmap.yaml | 29 ++++ manifests/zcloud-prod3/kustomization.yaml | 2 + manifests/zing-preview/configmap.yaml | 5 +- .../sharing/server/DeltaSharingService.scala | 125 ++++++++++++++++-- .../server/telemetry/AccessLogEmitter.scala | 75 +++++++++++ .../server/telemetry/GcpPricingTier.scala | 82 ++++++++++-- .../server/DeltaSharingServiceSuite.scala | 8 +- 12 files changed, 329 insertions(+), 43 deletions(-) create mode 100644 manifests/zcloud-emea/configmap.yaml create mode 100644 manifests/zcloud-prod3/configmap.yaml diff --git a/manifests/base/configmap.yaml b/manifests/base/configmap.yaml index b6055ad20..e0112b525 100644 --- a/manifests/base/configmap.yaml +++ b/manifests/base/configmap.yaml @@ -28,8 +28,7 @@ data: refreshTokenTtlMs: 3600000 accessLogging: enabled: true + sourceRegion: "us-central1" + detectGcpTraffic: true clientRegionHeader: "x-client-region" - clientRegionSubdivisionHeader: "x-client-region-subdivision" clientIpHeader: "x-forwarded-for" - pricingGroups: {} - defaultPricingGroup: "unknown" diff --git a/manifests/zcloud-emea/configmap.yaml b/manifests/zcloud-emea/configmap.yaml new file mode 100644 index 000000000..ca48f1c4e --- /dev/null +++ b/manifests/zcloud-emea/configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: delta-sharing-base-config + namespace: default +data: + base-config.yaml: | + version: 1 + authorization: + bearerToken: "BEARER_TOKEN_PLACEHOLDER" + host: "0.0.0.0" + port: 8080 + endpoint: "/delta-sharing" + preSignedUrlTimeoutSeconds: 3600 + deltaTableCacheSize: 100 + stalenessAcceptable: false + evaluatePredicateHints: false + evaluateJsonPredicateHints: true + evaluateJsonPredicateHintsV2: true + requestTimeoutSeconds: 180 + queryTablePageSizeLimit: 10000 + queryTablePageTokenTtlMs: 259200000 + refreshTokenTtlMs: 3600000 + accessLogging: + enabled: true + sourceRegion: "europe-west3" + detectGcpTraffic: true + clientRegionHeader: "x-client-region" + clientIpHeader: "x-forwarded-for" diff --git a/manifests/zcloud-emea/kustomization.yaml b/manifests/zcloud-emea/kustomization.yaml index 27bb5cb0b..cbec7e795 100644 --- a/manifests/zcloud-emea/kustomization.yaml +++ b/manifests/zcloud-emea/kustomization.yaml @@ -2,3 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base +patchesStrategicMerge: + - configmap.yaml diff --git a/manifests/zcloud-prod/configmap.yaml b/manifests/zcloud-prod/configmap.yaml index 983ce87d0..155fd5738 100644 --- a/manifests/zcloud-prod/configmap.yaml +++ b/manifests/zcloud-prod/configmap.yaml @@ -23,8 +23,7 @@ data: refreshTokenTtlMs: 3600000 accessLogging: enabled: true + sourceRegion: "us-central1" + detectGcpTraffic: true clientRegionHeader: "x-client-region" - clientRegionSubdivisionHeader: "x-client-region-subdivision" clientIpHeader: "x-forwarded-for" - pricingGroups: {} - defaultPricingGroup: "unknown" diff --git a/manifests/zcloud-prod2/configmap.yaml b/manifests/zcloud-prod2/configmap.yaml index 983ce87d0..577806dd9 100644 --- a/manifests/zcloud-prod2/configmap.yaml +++ b/manifests/zcloud-prod2/configmap.yaml @@ -23,8 +23,7 @@ data: refreshTokenTtlMs: 3600000 accessLogging: enabled: true + sourceRegion: "us-west4" + detectGcpTraffic: true clientRegionHeader: "x-client-region" - clientRegionSubdivisionHeader: "x-client-region-subdivision" clientIpHeader: "x-forwarded-for" - pricingGroups: {} - defaultPricingGroup: "unknown" diff --git a/manifests/zcloud-prod3/configmap.yaml b/manifests/zcloud-prod3/configmap.yaml new file mode 100644 index 000000000..62b112646 --- /dev/null +++ b/manifests/zcloud-prod3/configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: delta-sharing-base-config + namespace: default +data: + base-config.yaml: | + version: 1 + authorization: + bearerToken: "BEARER_TOKEN_PLACEHOLDER" + host: "0.0.0.0" + port: 8080 + endpoint: "/delta-sharing" + preSignedUrlTimeoutSeconds: 3600 + deltaTableCacheSize: 100 + stalenessAcceptable: false + evaluatePredicateHints: false + evaluateJsonPredicateHints: true + evaluateJsonPredicateHintsV2: true + requestTimeoutSeconds: 180 + queryTablePageSizeLimit: 10000 + queryTablePageTokenTtlMs: 259200000 + refreshTokenTtlMs: 3600000 + accessLogging: + enabled: true + sourceRegion: "australia-southeast1" + detectGcpTraffic: true + clientRegionHeader: "x-client-region" + clientIpHeader: "x-forwarded-for" diff --git a/manifests/zcloud-prod3/kustomization.yaml b/manifests/zcloud-prod3/kustomization.yaml index 27bb5cb0b..cbec7e795 100644 --- a/manifests/zcloud-prod3/kustomization.yaml +++ b/manifests/zcloud-prod3/kustomization.yaml @@ -2,3 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base +patchesStrategicMerge: + - configmap.yaml diff --git a/manifests/zing-preview/configmap.yaml b/manifests/zing-preview/configmap.yaml index 983ce87d0..155fd5738 100644 --- a/manifests/zing-preview/configmap.yaml +++ b/manifests/zing-preview/configmap.yaml @@ -23,8 +23,7 @@ data: refreshTokenTtlMs: 3600000 accessLogging: enabled: true + sourceRegion: "us-central1" + detectGcpTraffic: true clientRegionHeader: "x-client-region" - clientRegionSubdivisionHeader: "x-client-region-subdivision" clientIpHeader: "x-forwarded-for" - pricingGroups: {} - defaultPricingGroup: "unknown" diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 85204afde..146f219c6 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -49,7 +49,9 @@ import io.delta.sharing.server.common.JsonUtils import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ -import io.delta.sharing.server.telemetry.{AccessLogEmitter, AccessLogEntry, GcpPricingTier} +import io.delta.sharing.server.telemetry.{ + AccessLogEmitter, AccessLogEntry, GcpPricingTier, PricingContextLogEntry +} object ErrorCode { val UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -207,22 +209,42 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() - val location = DeltaSharingService.buildClientLocationContext( + val pricingCtx = DeltaSharingService.buildPricingContext( req, serverConfig.getAccessLogging) - // Emit to access log + // Emit access log with essential fields val entry = AccessLogEntry( share = share, schema = schema, table = table, egressBytes = bytes, timestampMs = nowMs, - pricingTier = location.pricingTier, - clientRegion = location.clientRegion, + pricingTier = pricingCtx.location.pricingTier, + clientRegion = pricingCtx.location.clientRegion, requestType = AccessLogEmitter.QueryRequestType ) accessLogEmitter.record(entry) + + // Emit context log with full details for verification + val contextEntry = PricingContextLogEntry( + share = share, + table = table, + timestampMs = nowMs, + clientIp = pricingCtx.clientIp, + clientIpSource = pricingCtx.clientIpSource, + rawRegionHeader = pricingCtx.rawRegionHeader, + regionHeaderSource = pricingCtx.regionHeaderSource, + hasEnvoyMetadata = pricingCtx.hasEnvoyMetadata, + isGcpIp = pricingCtx.isGcpIp, + egressType = pricingCtx.egressType, + sourceRegion = pricingCtx.sourceRegion, + sourceContinent = pricingCtx.sourceContinent, + destinationRegion = pricingCtx.destinationRegion, + destinationContinent = pricingCtx.destinationContinent, + pricingTier = pricingCtx.location.pricingTier + ) + accessLogEmitter.recordContext(contextEntry) } private def emitCdfEgressMetric( @@ -237,22 +259,42 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() - val location = DeltaSharingService.buildClientLocationContext( + val pricingCtx = DeltaSharingService.buildPricingContext( req, serverConfig.getAccessLogging) - // Emit to access log + // Emit access log with essential fields val entry = AccessLogEntry( share = share, schema = schema, table = table, egressBytes = bytes, timestampMs = nowMs, - pricingTier = location.pricingTier, - clientRegion = location.clientRegion, + pricingTier = pricingCtx.location.pricingTier, + clientRegion = pricingCtx.location.clientRegion, requestType = AccessLogEmitter.CdfStreamRequestType ) accessLogEmitter.record(entry) + + // Emit context log with full details for verification + val contextEntry = PricingContextLogEntry( + share = share, + table = table, + timestampMs = nowMs, + clientIp = pricingCtx.clientIp, + clientIpSource = pricingCtx.clientIpSource, + rawRegionHeader = pricingCtx.rawRegionHeader, + regionHeaderSource = pricingCtx.regionHeaderSource, + hasEnvoyMetadata = pricingCtx.hasEnvoyMetadata, + isGcpIp = pricingCtx.isGcpIp, + egressType = pricingCtx.egressType, + sourceRegion = pricingCtx.sourceRegion, + sourceContinent = pricingCtx.sourceContinent, + destinationRegion = pricingCtx.destinationRegion, + destinationContinent = pricingCtx.destinationContinent, + pricingTier = pricingCtx.location.pricingTier + ) + accessLogEmitter.recordContext(contextEntry) } private def logCdfRequestComplete( @@ -875,6 +917,23 @@ object DeltaSharingService { clientRegion: Option[String], pricingTier: String) + /** + * Full context for pricing tier calculation, used for debugging/verification. + */ + private[server] case class PricingContext( + location: ClientLocationContext, + clientIp: Option[String], + clientIpSource: Option[String], + rawRegionHeader: Option[String], + regionHeaderSource: Option[String], + hasEnvoyMetadata: Boolean, + isGcpIp: Boolean, + egressType: String, + sourceRegion: Option[String], + sourceContinent: Option[String], + destinationRegion: Option[String], + destinationContinent: Option[String]) + private[server] def extractEgressBytes(actions: Seq[Object]): Long = { actions.map(extractActionBytes).sum } @@ -882,6 +941,12 @@ object DeltaSharingService { private[server] def buildClientLocationContext( req: HttpRequest, accessLoggingConfig: AccessLoggingConfig): ClientLocationContext = { + buildPricingContext(req, accessLoggingConfig).location + } + + private[server] def buildPricingContext( + req: HttpRequest, + accessLoggingConfig: AccessLoggingConfig): PricingContext = { val headerMap: Map[String, String] = req.headers().names().asScala .flatMap { name => Option(req.headers().get(name)).map { value => @@ -890,12 +955,18 @@ object DeltaSharingService { } .toMap - buildClientLocationContext(headerMap, accessLoggingConfig) + buildPricingContext(headerMap, accessLoggingConfig) } private[server] def buildClientLocationContext( requestHeaders: Map[String, String], accessLoggingConfig: AccessLoggingConfig): ClientLocationContext = { + buildPricingContext(requestHeaders, accessLoggingConfig).location + } + + private[server] def buildPricingContext( + requestHeaders: Map[String, String], + accessLoggingConfig: AccessLoggingConfig): PricingContext = { val cfgOpt = Option(accessLoggingConfig) val normalizedHeaders = requestHeaders.map { case (k, v) => k.toLowerCase(Locale.ROOT) -> v @@ -913,7 +984,15 @@ object DeltaSharingService { val ipWithSource = getFirstClientIp(ipHeaders, normalizedHeaders) val region = regionWithSource.map(_._1) + val rawRegion = regionWithSource.map { case (normalized, header) => + normalizedHeaders.getOrElse(header, normalized) + } + val regionSource = regionWithSource.map(_._2) val clientIp = ipWithSource.map(_._1) + val clientIpSource = ipWithSource.map(_._2) + + // Check if client IP is in GCP public ranges + val isGcpIp = clientIp.exists(GcpPricingTier.isGcpPublicIp) // GCP Pricing Tier calculation val sourceRegion = cfgOpt @@ -933,6 +1012,11 @@ object DeltaSharingService { region, detectGcpTraffic) + // Calculate continents for context logging + val sourceContinent = sourceRegion.map(GcpPricingTier.continentFromGcpRegion) + val destContinent = destGcpRegion.map(GcpPricingTier.continentFromGcpRegion) + .orElse(region.map(GcpPricingTier.continentFromCountryCode)) + // Calculate pricing tier: // - For internet traffic: based on destination only (no sourceRegion needed) // - For inter-region traffic: requires sourceRegion @@ -947,13 +1031,28 @@ object DeltaSharingService { case GcpPricingTier.EgressType.INTER_REGION if sourceRegion.isDefined => GcpPricingTier.calculatePricingTier( sourceRegion, destGcpRegion, region, egressType) + case GcpPricingTier.EgressType.INTER_REGION => + // Inter-region detected but no source region configured - use placeholder + "interregion_unknown" case _ => "unknown" } - ClientLocationContext( - clientRegion = region, - pricingTier = pricingTier) + PricingContext( + location = ClientLocationContext( + clientRegion = region, + pricingTier = pricingTier), + clientIp = clientIp, + clientIpSource = clientIpSource, + rawRegionHeader = rawRegion, + regionHeaderSource = regionSource, + hasEnvoyMetadata = envoyPeerMetadata.isDefined, + isGcpIp = isGcpIp, + egressType = egressType.toString, + sourceRegion = sourceRegion, + sourceContinent = sourceContinent.map(_.toString), + destinationRegion = destGcpRegion, + destinationContinent = destContinent.map(_.toString)) } private def getOrderedHeaders( diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index fb0dd4b74..74348bbc8 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -71,12 +71,56 @@ case class AccessLogEntry( clientRegion: Option[String] = None, requestType: String = "query") +/** + * Captures all context information used to calculate the pricing tier. + * Emitted alongside ACCESS_LOG for debugging and accuracy verification. + * + * @param share The share being accessed (for correlation) + * @param table The table being accessed (for correlation) + * @param timestampMs Timestamp for correlation with ACCESS_LOG + * + * == Raw Header Values == + * @param clientIp First non-private, non-GCP IP from X-Forwarded-For chain + * @param clientIpSource Header that provided the client IP (e.g., "x-forwarded-for") + * @param rawRegionHeader Raw value from region header (before normalization) + * @param regionHeaderSource Header that provided the region (e.g., "x-client-region") + * @param hasEnvoyMetadata Whether X-Envoy-Peer-Metadata header was present + * @param isGcpIp Whether the client IP is in a known GCP public IP range (34.x, 35.x) + * + * == Derived Values == + * @param egressType Detected egress type: internet, inter_region, same_region, same_zone + * @param sourceRegion GCP region where server runs (from config) + * @param sourceContinent Continent of source region + * @param destinationRegion GCP region of destination (if detected from Envoy metadata) + * @param destinationContinent Continent of destination + * @param pricingTier Final calculated pricing tier + */ +case class PricingContextLogEntry( + share: String, + table: String, + timestampMs: Long, + // Raw header values + clientIp: Option[String] = None, + clientIpSource: Option[String] = None, + rawRegionHeader: Option[String] = None, + regionHeaderSource: Option[String] = None, + hasEnvoyMetadata: Boolean = false, + isGcpIp: Boolean = false, + // Derived values + egressType: String = "unknown", + sourceRegion: Option[String] = None, + sourceContinent: Option[String] = None, + destinationRegion: Option[String] = None, + destinationContinent: Option[String] = None, + pricingTier: String = "unknown") + /** * Trait for emitting access logs for share data egress tracking. * Implementations can write to different destinations (logging, external services, etc.) */ trait AccessLogEmitter { def record(entry: AccessLogEntry): Unit + def recordContext(entry: PricingContextLogEntry): Unit } object AccessLogEmitter { @@ -101,6 +145,7 @@ object AccessLogEmitter { */ object NoopAccessLogEmitter extends AccessLogEmitter { override def record(entry: AccessLogEntry): Unit = {} + override def recordContext(entry: PricingContextLogEntry): Unit = {} } /** @@ -137,4 +182,34 @@ class JsonAccessLogEmitter extends AccessLogEmitter { logger.info(JsonUtils.toJson(logPayload)) } + + override def recordContext(entry: PricingContextLogEntry): Unit = { + // Core payload for correlation + val basePayload = Map( + "logType" -> "PRICING_CONTEXT", + "share" -> entry.share, + "table" -> entry.table, + "timestampMs" -> entry.timestampMs, + "egressType" -> entry.egressType, + "pricingTier" -> entry.pricingTier, + "hasEnvoyMetadata" -> entry.hasEnvoyMetadata, + "isGcpIp" -> entry.isGcpIp + ) + + // Optional context fields - only include if present + val optionalPayload = Seq( + entry.clientIp.map("clientIp" -> _), + entry.clientIpSource.map("clientIpSource" -> _), + entry.rawRegionHeader.map("rawRegionHeader" -> _), + entry.regionHeaderSource.map("regionHeaderSource" -> _), + entry.sourceRegion.map("sourceRegion" -> _), + entry.sourceContinent.map("sourceContinent" -> _), + entry.destinationRegion.map("destinationRegion" -> _), + entry.destinationContinent.map("destinationContinent" -> _) + ).flatten.toMap + + val logPayload = basePayload ++ optionalPayload + + logger.info(JsonUtils.toJson(logPayload)) + } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala index 7dee2cbd9..c12be581a 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -273,23 +273,52 @@ object GcpPricingTier { def isLikelySameClusterTraffic(clientIp: Option[String]): Boolean = { clientIp match { case None => true // No forwarding header often means same-cluster - case Some(ip) => - // Check for common private IP ranges used in Kubernetes - ip.startsWith("10.") || - ip.startsWith("172.16.") || ip.startsWith("172.17.") || ip.startsWith("172.18.") || - ip.startsWith("172.19.") || ip.startsWith("172.20.") || ip.startsWith("172.21.") || - ip.startsWith("172.22.") || ip.startsWith("172.23.") || ip.startsWith("172.24.") || - ip.startsWith("172.25.") || ip.startsWith("172.26.") || ip.startsWith("172.27.") || - ip.startsWith("172.28.") || ip.startsWith("172.29.") || ip.startsWith("172.30.") || - ip.startsWith("172.31.") || - ip.startsWith("192.168.") || - ip == "127.0.0.1" || ip == "::1" + case Some(ip) => isPrivateIp(ip) } } + /** + * Check if an IP address is in a private/internal range. + */ + def isPrivateIp(ip: String): Boolean = { + ip.startsWith("10.") || + ip.startsWith("172.16.") || ip.startsWith("172.17.") || ip.startsWith("172.18.") || + ip.startsWith("172.19.") || ip.startsWith("172.20.") || ip.startsWith("172.21.") || + ip.startsWith("172.22.") || ip.startsWith("172.23.") || ip.startsWith("172.24.") || + ip.startsWith("172.25.") || ip.startsWith("172.26.") || ip.startsWith("172.27.") || + ip.startsWith("172.28.") || ip.startsWith("172.29.") || ip.startsWith("172.30.") || + ip.startsWith("172.31.") || + ip.startsWith("192.168.") || + ip == "127.0.0.1" || ip == "::1" + } + + /** + * Check if an IP address is in GCP's public IP ranges. + * GCP commonly uses these ranges for Cloud NAT, GKE nodes, and other services. + * + * Note: This is a heuristic. For precise detection, GCP publishes their IP ranges at: + * https://www.gstatic.com/ipranges/cloud.json + * + * @param ip The IP address to check + * @return true if the IP appears to be a GCP public IP + */ + def isGcpPublicIp(ip: String): Boolean = { + // GCP commonly uses 34.x.x.x and 35.x.x.x ranges + // These are the most common ranges for GKE, Cloud Run, Compute Engine, etc. + ip.startsWith("34.") || ip.startsWith("35.") + } + /** * Determine the egress type based on available information. * + * Detection priority: + * 1. Private IP (10.x, 172.16-31.x, 192.168.x) => SAME_REGION (internal cluster) + * 2. Envoy metadata with GCP region extractable => INTER_REGION + * 3. Envoy metadata present + GCP public IP => INTER_REGION (service mesh traffic) + * 4. X-Client-Region = "ZZ" => INTER_REGION (GCP internal) + * 5. Valid country code in X-Client-Region => INTERNET + * 6. Otherwise => UNKNOWN + * * @param clientIp The client IP address (from X-Forwarded-For) * @param envoyPeerMetadata The X-Envoy-Peer-Metadata header value * @param clientRegion The client region from X-Client-Region header @@ -302,20 +331,45 @@ object GcpPricingTier { clientRegion: Option[String], detectGcpTraffic: Boolean): (EgressType, Option[String]) = { - // Check for same-cluster traffic first + // Check for same-cluster traffic first (private IPs) if (isLikelySameClusterTraffic(clientIp)) { return (SAME_REGION, None) } - // If GCP traffic detection is enabled, try to extract region from Envoy metadata + // If GCP traffic detection is enabled if (detectGcpTraffic) { + // Try to extract specific GCP region from Envoy metadata val gcpRegion = envoyPeerMetadata.flatMap(extractGcpRegionFromEnvoyMetadata) if (gcpRegion.isDefined) { return (INTER_REGION, gcpRegion) } + + // If Envoy metadata is present (service mesh) AND client IP is GCP public range, + // this is inter-region GCP traffic even if we can't extract the specific region + val hasEnvoyMetadata = envoyPeerMetadata.isDefined + val isGcpIp = clientIp.exists(isGcpPublicIp) + if (hasEnvoyMetadata && isGcpIp) { + return (INTER_REGION, None) + } + + // If client IP is GCP range but no Envoy metadata, still likely GCP traffic + // (e.g., Cloud Run, Cloud Functions calling directly) + if (isGcpIp) { + // Check if region header suggests internal traffic + clientRegion match { + // scalastyle:off caselocale + case Some(region) if region.toUpperCase == "ZZ" => + // scalastyle:on caselocale + return (INTER_REGION, None) + case _ => + // GCP IP but valid country code - could be GCP service with external IP + // Classify as inter-region since it's still GCP-to-GCP + return (INTER_REGION, None) + } + } } - // Check if X-Client-Region indicates unknown/internal (ZZ) + // Check X-Client-Region for remaining cases clientRegion match { // scalastyle:off caselocale case Some(region) if region.toUpperCase == "ZZ" => diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index fb46330aa..991d95548 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -405,7 +405,7 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { assert(ctx.pricingTier == "internet_to_na_eu") } - test("buildClientLocationContext returns unknown for inter-region without source") { + test("buildClientLocationContext returns interregion_unknown for GCP IP without source") { val cfg = new AccessLoggingConfig() cfg.setEnabled(true) cfg.setDetectGcpTraffic(true) @@ -414,12 +414,12 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { val ctx = DeltaSharingService.buildClientLocationContext( Map( "X-Client-Region" -> "ZZ", // ZZ indicates internal GCP traffic - "X-Forwarded-For" -> "34.100.0.1" + "X-Forwarded-For" -> "34.100.0.1" // GCP public IP ), cfg) - // Inter-region requires sourceRegion, so unknown pricing tier - assert(ctx.pricingTier == "unknown") + // Inter-region detected (GCP IP + ZZ region), but no sourceRegion for exact tier + assert(ctx.pricingTier == "interregion_unknown") } integrationTest("401 Unauthorized Error: incorrect token") { From 237fbf02be95b95515f40591de765bae82b30767 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Thu, 4 Jun 2026 11:26:23 +0300 Subject: [PATCH 09/15] Fix interregion handling --- .../io/delta/sharing/server/telemetry/GcpPricingTier.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala index c12be581a..d5ccb1643 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -175,8 +175,11 @@ object GcpPricingTier { case INTER_REGION => val srcContinent = sourceRegion .map(continentFromGcpRegion).getOrElse(UNKNOWN_CONTINENT) + // For inter-region, try GCP region first, fall back to country code val dstContinent = destinationRegion - .map(continentFromGcpRegion).getOrElse(UNKNOWN_CONTINENT) + .map(continentFromGcpRegion) + .orElse(destinationCountry.map(continentFromCountryCode)) + .getOrElse(UNKNOWN_CONTINENT) calculateInterRegionTier(srcContinent, dstContinent) case INTERNET => From 56435d5c596859640f8905f660acf8ed270f9be6 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Thu, 4 Jun 2026 16:11:53 +0300 Subject: [PATCH 10/15] Fix same region resolvment. Add doc --- docs/PER_SHARE_EGRESS_MONITORING.md | 304 ++++++++++++++++++ .../sharing/server/DeltaSharingService.scala | 39 ++- .../server/telemetry/AccessLogEmitter.scala | 29 ++ .../server/telemetry/GcpPricingTier.scala | 47 ++- .../telemetry/GcpPricingTierSuite.scala | 79 +++++ 5 files changed, 486 insertions(+), 12 deletions(-) create mode 100644 docs/PER_SHARE_EGRESS_MONITORING.md diff --git a/docs/PER_SHARE_EGRESS_MONITORING.md b/docs/PER_SHARE_EGRESS_MONITORING.md new file mode 100644 index 000000000..4f7aaebef --- /dev/null +++ b/docs/PER_SHARE_EGRESS_MONITORING.md @@ -0,0 +1,304 @@ +# Per-Share Egress Monitoring with Pricing Group Determination + +## Overview + +This document describes the new per-share egress monitoring feature introduced in the ZING-43690 branch. The feature provides: + +1. **Share-attributed egress tracking** — Structured access logs for every query and CDF request +2. **GCP pricing tier classification** — Automatic determination of egress cost categories +3. **Traffic type detection** — Distinguishing between internet, inter-region GCP, and same-region traffic + +## Feature Summary + +When enabled, the Delta Sharing Server emits structured JSON log entries for each data access request. These logs include: + +- **Share identification**: share name, schema, table +- **Data volume**: total egress bytes transferred +- **Pricing tier**: GCP egress pricing category (e.g., `internet_to_na_eu`, `interregion_na_to_na`) +- **Client context**: region code, request type + +--- + +## Pricing Group Types + +The system classifies egress traffic into pricing tiers based on GCP's network pricing structure. There are three main categories: + +### 1. Free/Internal Traffic + +| Tier | Description | Approximate Cost | +|------|-------------|------------------| +| `same_zone` | Traffic within the same GCP zone | Free | +| `same_region` | Traffic within the same region or Kubernetes cluster | ~Free | + +### 2. Inter-Region GCP Traffic + +Traffic between GCP services (e.g., between regions via service mesh): + +| Tier | Route | Approximate Cost | +|------|-------|------------------| +| `interregion_na_to_na` | North America → North America | $0.02/GiB | +| `interregion_eu_to_eu` | Europe → Europe | $0.02/GiB | +| `interregion_na_to_eu` | North America → Europe | $0.05/GiB | +| `interregion_eu_to_na` | Europe → North America | $0.05/GiB | +| `interregion_to_apac` | Any region → Asia Pacific | $0.08/GiB | +| `interregion_to_oceania` | Any region → Australia/Oceania | $0.10/GiB | +| `interregion_to_latam` | Any region → Latin America | $0.14/GiB | + +### 3. Internet Egress (Premium Tier) + +Traffic leaving GCP to external clients: + +| Tier | Destination | Approximate Cost | +|------|-------------|------------------| +| `internet_to_na_eu` | North America or Europe | $0.12/GiB | +| `internet_to_apac` | Asia Pacific | $0.12/GiB | +| `internet_to_latam` | Latin America | $0.19/GiB | +| `internet_to_oceania` | Australia/Oceania | $0.15/GiB | + +### Special Cases + +| Tier | Description | +|------|-------------| +| `interregion_unknown` | Inter-region traffic detected but source region not configured | +| `unknown` | Unable to determine pricing tier | + +--- + +## How Pricing Groups Are Resolved + +The pricing tier is determined through a multi-step detection process: + +### Step 1: Determine Egress Type + +The system first classifies the traffic type based on available signals: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Egress Type Detection │ +├─────────────────────────────────────────────────────────────────┤ +│ 1. Client IP is private (10.x, 172.16-31.x, 192.168.x)? │ +│ → SAME_REGION (internal cluster traffic) │ +│ │ +│ 2. GCP detection enabled AND Envoy metadata contains region? │ +│ a. Extracted client region matches configured sourceRegion? │ +│ → SAME_REGION (same-region GCP traffic) │ +│ b. Otherwise │ +│ → INTER_REGION (with specific GCP region) │ +│ │ +│ 3. Envoy metadata present AND client IP is GCP range (34.x/35.x)?│ +│ → INTER_REGION (service mesh traffic) │ +│ │ +│ 4. Client IP is GCP range without Envoy metadata? │ +│ → INTER_REGION (GCP service calling directly) │ +│ │ +│ 5. X-Client-Region header = "ZZ"? │ +│ → INTER_REGION (GCP internal, location unknown) │ +│ │ +│ 6. X-Client-Region contains valid country code? │ +│ → INTERNET (external client) │ +│ │ +│ 7. No region header present? │ +│ → UNKNOWN │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Step 2: Determine Continents + +Based on the source region (server configuration) and destination (detected from headers): + +**GCP Regions → Continent Mapping:** +- `us-*`, `northamerica-*` → North America (NA) +- `europe-*` → Europe (EU) +- `asia-*` → Asia Pacific (APAC) +- `southamerica-*` → Latin America (LATAM) +- `australia-*` → Oceania (OCEANIA) + +**Country Codes → Continent Mapping:** +- US, CA, MX → NA +- GB, IE, DE, FR, NL, BE, CH, AT, ES, PT, IT, SE, NO, DK, FI, PL, CZ, HU, RO, etc. → EU +- JP, KR, CN, HK, TW, SG, MY, ID, TH, VN, PH, IN, PK, BD → APAC +- AU, NZ → OCEANIA +- BR, AR, CL, CO, PE, VE, EC, etc. → LATAM + +### Step 3: Calculate Pricing Tier + +Based on egress type and continent pair: + +| Egress Type | Calculation Method | +|-------------|-------------------| +| `SAME_ZONE` | Returns `same_zone` | +| `SAME_REGION` | Returns `same_region` | +| `INTER_REGION` | Based on source→destination continent pair | +| `INTERNET` | Based on destination continent only | + +--- + +## Configuration + +Enable access logging in your server configuration: + +```yaml +accessLogging: + # Enable/disable the feature + enabled: true + + # GCP region where this server runs (required for inter-region pricing) + sourceRegion: "us-central1" + + # Enable GCP inter-region traffic detection + detectGcpTraffic: true + + # Header containing client country code (set via load balancer) + clientRegionHeader: "x-client-region" + + # Header containing client IP chain + clientIpHeader: "x-forwarded-for" + + # Optional: custom pricing group mappings + pricingGroups: {} + + # Fallback when no location can be resolved + defaultPricingGroup: "unknown" +``` + +### GCP Load Balancer Configuration + +To populate the `X-Client-Region` header, configure your GCP load balancer backend with custom request headers: + +``` +X-Client-Region: {client_region} +X-Client-Region-Subdivision: {client_region_subdivision} +``` + +--- + +## Log Output + +### ACCESS_LOG Entry + +Emitted for each query/CDF request with non-zero egress: + +```json +{ + "logType": "ACCESS_LOG", + "share": "myshare", + "schema": "myschema", + "table": "mytable", + "egressBytes": 1048576, + "pricingTier": "internet_to_na_eu", + "timestampMs": 1717502400000, + "requestType": "query", + "clientRegion": "US" +} +``` + +### PRICING_CONTEXT Entry + +Emitted alongside ACCESS_LOG for debugging and verification: + +```json +{ + "logType": "PRICING_CONTEXT", + "share": "myshare", + "table": "mytable", + "timestampMs": 1717502400000, + "clientIp": "203.0.113.45", + "clientIpSource": "x-forwarded-for", + "rawRegionHeader": "US", + "regionHeaderSource": "x-client-region", + "hasEnvoyMetadata": false, + "isGcpIp": false, + "egressType": "internet", + "sourceRegion": "us-central1", + "sourceContinent": "NA", + "destinationContinent": "NA", + "pricingTier": "internet_to_na_eu" +} +``` + +### REQUEST_HEADERS Entry + +Emitted alongside ACCESS_LOG and PRICING_CONTEXT for debugging header detection issues: + +```json +{ + "logType": "REQUEST_HEADERS", + "share": "myshare", + "table": "mytable", + "timestampMs": 1717502400000, + "headers": { + "x-forwarded-for": "34.45.22.184", + "x-client-region": "US", + "x-envoy-peer-metadata": "eyJnY3BfbG9jYXRpb24iOiJ1cy1jZW50cmFsMS1mIn0=", + "content-type": "application/json", + "authorization": "Bearer ***" + } +} +``` + +This log entry contains all request headers (lowercased) and is useful for debugging +why a particular pricing tier was assigned. It helps verify whether the expected +headers (like `x-envoy-peer-metadata` with `gcp_location`) are being passed correctly. + +--- + +## Header Detection Priority + +The system checks multiple headers in priority order: + +**Region Headers:** +1. Configured `clientRegionHeader` (default: `x-client-region`) +2. `x-appengine-country` +3. `cf-ipcountry` +4. `cloudfront-viewer-country` + +**Client IP Headers:** +1. Configured `clientIpHeader` (default: `x-forwarded-for`) +2. `x-envoy-external-address` +3. `x-real-ip` +4. `true-client-ip` + +For IP chains (comma-separated), the first public IP is selected. + +--- + +## Implementation Details + +### Key Components + +| File | Purpose | +|------|---------| +| `GcpPricingTier.scala` | Continent mapping, egress type detection, pricing tier calculation | +| `AccessLogEmitter.scala` | Log entry models and JSON emission | +| `DeltaSharingService.scala` | Integration points for query and CDF endpoints | +| `ServerConfig.scala` | `AccessLoggingConfig` configuration model | + +### Egress Bytes Calculation + +Egress bytes are calculated by summing the `size` field from all file actions in the response: +- `AddFile` — Regular query results +- `AddFileForCDF` — CDF add file actions +- `AddCDCFile` — CDF change data capture files + +--- + +## Deployment Configurations + +The feature is enabled in production environments with the following settings: + +**zcloud-prod (us-central1):** +```yaml +accessLogging: + enabled: true + sourceRegion: "us-central1" + detectGcpTraffic: true +``` + +--- + +## Notes + +- Logs are emitted to a dedicated logger (`delta.sharing.access`) for easy filtering in Cloud Logging +- Zero-byte requests are not logged +- The `PRICING_CONTEXT` log provides full audit trail for pricing decisions +- Pricing tier strings match GCP documentation for easier cost analysis diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 146f219c6..0c8d5552c 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -50,7 +50,7 @@ import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model.{AddCDCFile, AddFile, AddFileForCDF, QueryStatus, RemoveFile, SingleAction} import io.delta.sharing.server.protocol._ import io.delta.sharing.server.telemetry.{ - AccessLogEmitter, AccessLogEntry, GcpPricingTier, PricingContextLogEntry + AccessLogEmitter, AccessLogEntry, GcpPricingTier, PricingContextLogEntry, RequestHeadersLogEntry } object ErrorCode { @@ -197,6 +197,16 @@ class DeltaSharingService(serverConfig: ServerConfig) { private val logger = LoggerFactory.getLogger(classOf[DeltaSharingService]) private val accessLogEmitter = AccessLogEmitter.create(serverConfig) + private def extractRequestHeaders(req: HttpRequest): Map[String, String] = { + req.headers().names().asScala + .flatMap { name => + Option(req.headers().get(name)).map { value => + name.toString.toLowerCase(Locale.ROOT) -> value + } + } + .toMap + } + private def emitQueryEgressMetric( req: HttpRequest, share: String, @@ -209,8 +219,9 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() + val headers = extractRequestHeaders(req) val pricingCtx = DeltaSharingService.buildPricingContext( - req, + headers, serverConfig.getAccessLogging) // Emit access log with essential fields @@ -245,6 +256,15 @@ class DeltaSharingService(serverConfig: ServerConfig) { pricingTier = pricingCtx.location.pricingTier ) accessLogEmitter.recordContext(contextEntry) + + // Emit all request headers for debugging pricing tier detection + val headersEntry = RequestHeadersLogEntry( + share = share, + table = table, + timestampMs = nowMs, + headers = headers + ) + accessLogEmitter.recordHeaders(headersEntry) } private def emitCdfEgressMetric( @@ -259,8 +279,9 @@ class DeltaSharingService(serverConfig: ServerConfig) { } val nowMs = System.currentTimeMillis() + val headers = extractRequestHeaders(req) val pricingCtx = DeltaSharingService.buildPricingContext( - req, + headers, serverConfig.getAccessLogging) // Emit access log with essential fields @@ -295,6 +316,15 @@ class DeltaSharingService(serverConfig: ServerConfig) { pricingTier = pricingCtx.location.pricingTier ) accessLogEmitter.recordContext(contextEntry) + + // Emit all request headers for debugging pricing tier detection + val headersEntry = RequestHeadersLogEntry( + share = share, + table = table, + timestampMs = nowMs, + headers = headers + ) + accessLogEmitter.recordHeaders(headersEntry) } private def logCdfRequestComplete( @@ -1010,7 +1040,8 @@ object DeltaSharingService { clientIp, envoyPeerMetadata, region, - detectGcpTraffic) + detectGcpTraffic, + sourceRegion) // Calculate continents for context logging val sourceContinent = sourceRegion.map(GcpPricingTier.continentFromGcpRegion) diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index 74348bbc8..212dd7853 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -114,6 +114,21 @@ case class PricingContextLogEntry( destinationContinent: Option[String] = None, pricingTier: String = "unknown") +/** + * Log entry containing all request headers for debugging pricing tier detection. + * Emitted alongside PRICING_CONTEXT when header debugging is needed. + * + * @param share Share name for correlation + * @param table Table name for correlation + * @param timestampMs Timestamp for correlation with ACCESS_LOG + * @param headers All request headers (keys lowercased) + */ +case class RequestHeadersLogEntry( + share: String, + table: String, + timestampMs: Long, + headers: Map[String, String]) + /** * Trait for emitting access logs for share data egress tracking. * Implementations can write to different destinations (logging, external services, etc.) @@ -121,6 +136,7 @@ case class PricingContextLogEntry( trait AccessLogEmitter { def record(entry: AccessLogEntry): Unit def recordContext(entry: PricingContextLogEntry): Unit + def recordHeaders(entry: RequestHeadersLogEntry): Unit } object AccessLogEmitter { @@ -146,6 +162,7 @@ object AccessLogEmitter { object NoopAccessLogEmitter extends AccessLogEmitter { override def record(entry: AccessLogEntry): Unit = {} override def recordContext(entry: PricingContextLogEntry): Unit = {} + override def recordHeaders(entry: RequestHeadersLogEntry): Unit = {} } /** @@ -212,4 +229,16 @@ class JsonAccessLogEmitter extends AccessLogEmitter { logger.info(JsonUtils.toJson(logPayload)) } + + override def recordHeaders(entry: RequestHeadersLogEntry): Unit = { + val logPayload = Map( + "logType" -> "REQUEST_HEADERS", + "share" -> entry.share, + "table" -> entry.table, + "timestampMs" -> entry.timestampMs, + "headers" -> entry.headers + ) + + logger.info(JsonUtils.toJson(logPayload)) + } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala index d5ccb1643..acf889a16 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -243,6 +243,11 @@ object GcpPricingTier { * The header is base64-encoded and contains structured metadata including * a "gcp_location" field with the region (e.g., "us-central1-f"). * + * The metadata can be in various formats: + * - JSON: {"gcp_location":"us-central1-f"} + * - Protobuf text: gcp_location:us-central1-f + * - Protobuf binary: gcp_location[binary bytes]us-central1-f + * * @param headerValue The base64-encoded X-Envoy-Peer-Metadata header value * @return Optional GCP region extracted from the metadata */ @@ -251,9 +256,14 @@ object GcpPricingTier { Try { val decoded = new String(Base64.getDecoder.decode(headerValue), "UTF-8") - // The metadata is typically in a protobuf-like format or JSON - // Look for patterns like "gcp_location":"us-central1-f" or gcp_location:us-central1-f - val gcpLocationPattern = """(?:gcp_location|GCP_LOCATION)["\s:]+([a-z]+-[a-z0-9-]+)""".r + // The metadata can be protobuf binary or text format. + // In protobuf binary, field names and values are separated by control characters. + // We look for "gcp_location" followed by any characters (including binary), + // then capture a GCP region pattern. + // The region format is: {continent}-{location}{number}[-{zone}] + // e.g., us-central1, europe-west1-b, asia-east1-a + val gcpLocationPattern = + """(?:gcp_location|GCP_LOCATION)[\x00-\x1f\s":]*([a-z]+-[a-z]+\d+(?:-[a-z])?)""".r gcpLocationPattern.findFirstMatchIn(decoded).map { m => // Extract region without zone suffix (e.g., "us-central1" from "us-central1-f") val fullLocation = m.group(1) @@ -316,7 +326,8 @@ object GcpPricingTier { * * Detection priority: * 1. Private IP (10.x, 172.16-31.x, 192.168.x) => SAME_REGION (internal cluster) - * 2. Envoy metadata with GCP region extractable => INTER_REGION + * 2. Envoy metadata with GCP region extractable => SAME_REGION if matches sourceRegion, + * otherwise INTER_REGION * 3. Envoy metadata present + GCP public IP => INTER_REGION (service mesh traffic) * 4. X-Client-Region = "ZZ" => INTER_REGION (GCP internal) * 5. Valid country code in X-Client-Region => INTERNET @@ -326,13 +337,15 @@ object GcpPricingTier { * @param envoyPeerMetadata The X-Envoy-Peer-Metadata header value * @param clientRegion The client region from X-Client-Region header * @param detectGcpTraffic Whether GCP traffic detection is enabled + * @param sourceRegion The GCP region where this server runs (for same-region detection) * @return Tuple of (EgressType, Optional destination GCP region) */ def determineEgressType( clientIp: Option[String], envoyPeerMetadata: Option[String], clientRegion: Option[String], - detectGcpTraffic: Boolean): (EgressType, Option[String]) = { + detectGcpTraffic: Boolean, + sourceRegion: Option[String] = None): (EgressType, Option[String]) = { // Check for same-cluster traffic first (private IPs) if (isLikelySameClusterTraffic(clientIp)) { @@ -342,9 +355,17 @@ object GcpPricingTier { // If GCP traffic detection is enabled if (detectGcpTraffic) { // Try to extract specific GCP region from Envoy metadata - val gcpRegion = envoyPeerMetadata.flatMap(extractGcpRegionFromEnvoyMetadata) - if (gcpRegion.isDefined) { - return (INTER_REGION, gcpRegion) + val clientGcpRegion = envoyPeerMetadata.flatMap(extractGcpRegionFromEnvoyMetadata) + if (clientGcpRegion.isDefined) { + // If we know both source and client regions, check if they match + val isSameRegion = (sourceRegion, clientGcpRegion) match { + case (Some(src), Some(dst)) => normalizeRegion(src) == normalizeRegion(dst) + case _ => false + } + if (isSameRegion) { + return (SAME_REGION, clientGcpRegion) + } + return (INTER_REGION, clientGcpRegion) } // If Envoy metadata is present (service mesh) AND client IP is GCP public range, @@ -387,4 +408,14 @@ object GcpPricingTier { (UNKNOWN_TYPE, None) } } + + /** + * Normalize a GCP region string for comparison. + * Removes zone suffix (e.g., "us-central1-f" -> "us-central1") and lowercases. + */ + private def normalizeRegion(region: String): String = { + val lower = region.toLowerCase(Locale.ROOT).trim + // Remove zone letter suffix if present (e.g., -a, -b, -f) + lower.replaceAll("-[a-z]$", "") + } } diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala index f99eba92b..d76918c50 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala @@ -251,6 +251,51 @@ class GcpPricingTierSuite extends FunSuite { assert(destRegion.isEmpty) } + test("determineEgressType returns SAME_REGION when client GCP region matches source region") { + // Create Envoy metadata with gcp_location + val payload = """{"gcp_location":"us-central1-f"}""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + + val (egressType, destRegion) = determineEgressType( + clientIp = Some("34.100.0.1"), + envoyPeerMetadata = Some(encoded), + clientRegion = Some("US"), + detectGcpTraffic = true, + sourceRegion = Some("us-central1")) + assert(egressType == SAME_REGION) + assert(destRegion.contains("us-central1")) + } + + test("determineEgressType returns INTER_REGION when client GCP region differs from source region") { + // Create Envoy metadata with gcp_location in different region + val payload = """{"gcp_location":"europe-west1-b"}""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + + val (egressType, destRegion) = determineEgressType( + clientIp = Some("34.100.0.1"), + envoyPeerMetadata = Some(encoded), + clientRegion = Some("DE"), + detectGcpTraffic = true, + sourceRegion = Some("us-central1")) + assert(egressType == INTER_REGION) + assert(destRegion.contains("europe-west1")) + } + + test("determineEgressType returns SAME_REGION with zone suffix variations") { + // Test that "us-central1" matches "us-central1-f" + val payload = """{"gcp_location":"us-central1-a"}""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + + val (egressType, destRegion) = determineEgressType( + clientIp = Some("34.100.0.1"), + envoyPeerMetadata = Some(encoded), + clientRegion = Some("US"), + detectGcpTraffic = true, + sourceRegion = Some("us-central1-f")) + assert(egressType == SAME_REGION) + assert(destRegion.contains("us-central1")) + } + // ===== Envoy Metadata Parsing Tests ===== test("extractGcpRegionFromEnvoyMetadata returns None for null/empty") { @@ -272,4 +317,38 @@ class GcpPricingTierSuite extends FunSuite { val result = extractGcpRegionFromEnvoyMetadata(encoded) assert(result.contains("europe-west1")) } + + test("extractGcpRegionFromEnvoyMetadata handles protobuf binary format") { + // Simulate protobuf binary where field name and value are separated by control chars + // gcp_location + [0x12 0x0f 0x1a 0x0d] + us-central1-f + val binaryPayload = "gcp_location\u0012\u000f\u001a\rus-central1-f" + val encoded = Base64.getEncoder.encodeToString(binaryPayload.getBytes("UTF-8")) + val result = extractGcpRegionFromEnvoyMetadata(encoded) + assert(result.contains("us-central1")) + } + + test("extractGcpRegionFromEnvoyMetadata parses real Istio/ASM metadata") { + // Real X-Envoy-Peer-Metadata from Istio ingress gateway (GKE with ASM) + // Contains PLATFORM_METADATA with gcp_location:us-central1-f + val realMetadata = "ChQKDkFQUF9DT05UQUlORVJTEgIaAAo9CgpDTFVTVEVSX0lEEi8aLWNuLXppbmct" + + "ZGV2LTE5NzUyMi11cy1jZW50cmFsMS1mLXppbmctY2x1c3RlcgoeCgxJTlNUQU5DRV9JUFMSDhoM" + + "MTAuMjUyLjguMTI3CiAKDUlTVElPX1ZFUlNJT04SDxoNMS4yMC44LWFzbS43MwqzAgoGTEFCRUxT" + + "EqgCKqUCCh0KA2FwcBIWGhRpc3Rpby1pbmdyZXNzZ2F0ZXdheQoZCgVpc3RpbxIQGg5pbmdyZXNz" + + "Z2F0ZXdheQodCgxpc3Rpby5pby9yZXYSDRoLYXNtLW1hbmFnZWQKOQofc2VydmljZS5pc3Rpby5p" + + "by9jYW5vbmljYWwtbmFtZRIWGhRpc3Rpby1pbmdyZXNzZ2F0ZXdheQovCiNzZXJ2aWNlLmlzdGlv" + + "LmlvL2Nhbm9uaWNhbC1yZXZpc2lvbhIIGgZsYXRlc3QKLgoddG9wb2xvZ3kua3ViZXJuZXRlcy5p" + + "by9yZWdpb24SDRoLdXMtY2VudHJhbDEKLgobdG9wb2xvZ3kua3ViZXJuZXRlcy5pby96b25lEg8a" + + "DXVzLWNlbnRyYWwxLWMKHgoHTUVTSF9JRBITGhFwcm9qLTMwMzkzMzg2ODgxMAovCgROQU1FEica" + + "JWlzdGlvLWluZ3Jlc3NnYXRld2F5LTY2NjRjZGRkN2YtamM5bTIKGwoJTkFNRVNQQUNFEg4aDGlz" + + "dGlvLXN5c3RlbQpdCgVPV05FUhJUGlJrdWJlcm5ldGVzOi8vYXBpcy9hcHBzL3YxL25hbWVzcGFj" + + "ZXMvaXN0aW8tc3lzdGVtL2RlcGxveW1lbnRzL2lzdGlvLWluZ3Jlc3NnYXRld2F5CrACChFQTEFU" + + "Rk9STV9NRVRBREFUQRKaAiqXAgomChRnY3BfZ2tlX2NsdXN0ZXJfbmFtZRIOGgx6aW5nLWNsdXN0" + + "ZXIKgwEKE2djcF9na2VfY2x1c3Rlcl91cmwSbBpqaHR0cHM6Ly9jb250YWluZXIuZ29vZ2xlYXBp" + + "cy5jb20vdjEvcHJvamVjdHMvemluZy1kZXYtMTk3NTIyL2xvY2F0aW9ucy91cy1jZW50cmFsMS1m" + + "L2NsdXN0ZXJzL3ppbmctY2x1c3RlcgofCgxnY3BfbG9jYXRpb24SDxoNdXMtY2VudHJhbDEtZgog" + + "CgtnY3BfcHJvamVjdBIRGg96aW5nLWRldi0xOTc1MjIKJAoSZ2NwX3Byb2plY3RfbnVtYmVyEg4a" + + "DDMwMzkzMzg2ODgxMAonCg1XT1JLTE9BRF9OQU1FEhYaFGlzdGlvLWluZ3Jlc3NnYXRld2F5" + val result = extractGcpRegionFromEnvoyMetadata(realMetadata) + assert(result.contains("us-central1"), s"Expected us-central1 but got $result") + } } From 37aebcdb740a118f4f63d3aad7c3182bf84422f5 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 8 Jun 2026 11:30:29 +0300 Subject: [PATCH 11/15] Add GCP IP region lookup --- .../server/telemetry/GcpIpRangeLookup.scala | 328 ++++++++++++++++++ .../server/telemetry/GcpPricingTier.scala | 90 +++-- .../telemetry/GcpIpRangeLookupSuite.scala | 246 +++++++++++++ .../telemetry/GcpPricingTierSuite.scala | 127 +++++-- 4 files changed, 722 insertions(+), 69 deletions(-) create mode 100644 server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala create mode 100644 server/src/test/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookupSuite.scala diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala new file mode 100644 index 000000000..ebda803f9 --- /dev/null +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala @@ -0,0 +1,328 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import java.net.{HttpURLConnection, InetAddress, URL} +import java.util.concurrent.{Executors, TimeUnit} +import java.util.concurrent.atomic.AtomicReference + +import scala.collection.mutable +import scala.util.{Failure, Success, Try} +import scala.util.control.NonFatal + +import io.delta.sharing.server.common.JsonUtils + +/** + * GCP IP Range lookup utility. + * + * Fetches GCP's published IP ranges from https://www.gstatic.com/ipranges/cloud.json + * and provides efficient IP-to-region lookup using a CIDR trie. + * + * The IP ranges are refreshed periodically (default: every 24 hours) to pick up + * any new allocations from GCP. + */ +object GcpIpRangeLookup { + + private val GCP_IP_RANGES_URL = "https://www.gstatic.com/ipranges/cloud.json" + private val REFRESH_INTERVAL_HOURS = 24 + private val CONNECTION_TIMEOUT_MS = 10000 + private val READ_TIMEOUT_MS = 30000 + + /** + * Case class representing a single IP prefix entry from GCP's JSON. + */ + case class IpPrefix( + ipv4Prefix: Option[String] = None, + ipv6Prefix: Option[String] = None, + service: String = "", + scope: String = "" + ) + + /** + * Case class representing the full GCP IP ranges response. + */ + case class GcpIpRanges( + syncToken: String = "", + creationTime: String = "", + prefixes: Seq[IpPrefix] = Seq.empty + ) + + /** + * A node in the CIDR trie for efficient IP-to-region lookup. + */ + private class CidrTrieNode { + var region: Option[String] = None + var children: Array[CidrTrieNode] = null + + def getOrCreateChild(bit: Int): CidrTrieNode = { + if (children == null) { + children = new Array[CidrTrieNode](2) + } + if (children(bit) == null) { + children(bit) = new CidrTrieNode() + } + children(bit) + } + + def getChild(bit: Int): Option[CidrTrieNode] = { + if (children == null) None + else Option(children(bit)) + } + } + + /** + * CIDR Trie for efficient longest-prefix-match lookups. + */ + private class CidrTrie { + private val root = new CidrTrieNode() + + /** + * Insert a CIDR block with its associated region. + */ + def insert(cidr: String, region: String): Unit = { + parseCidr(cidr).foreach { case (ipBytes, prefixLen) => + var node = root + for (i <- 0 until prefixLen) { + val byteIndex = i / 8 + val bitIndex = 7 - (i % 8) + val bit = (ipBytes(byteIndex) >> bitIndex) & 1 + node = node.getOrCreateChild(bit) + } + node.region = Some(region) + } + } + + /** + * Look up the region for an IP address using longest-prefix-match. + */ + def lookup(ip: String): Option[String] = { + parseIp(ip).flatMap { ipBytes => + var node = root + var lastMatch: Option[String] = None + var i = 0 + val maxBits = ipBytes.length * 8 + + while (i < maxBits && node != null) { + // Record the region if this node has one (longest prefix match) + if (node.region.isDefined) { + lastMatch = node.region + } + + val byteIndex = i / 8 + val bitIndex = 7 - (i % 8) + val bit = (ipBytes(byteIndex) >> bitIndex) & 1 + + node.getChild(bit) match { + case Some(child) => + node = child + i += 1 + case None => + node = null + } + } + + // Check if final node has a region + if (node != null && node.region.isDefined) { + lastMatch = node.region + } + + lastMatch + } + } + + /** + * Parse a CIDR notation string into (ip bytes, prefix length). + */ + private def parseCidr(cidr: String): Option[(Array[Byte], Int)] = { + Try { + val parts = cidr.split("/") + val ipBytes = InetAddress.getByName(parts(0)).getAddress + val prefixLen = parts(1).toInt + (ipBytes, prefixLen) + }.toOption + } + + /** + * Parse an IP address string into bytes. + */ + private def parseIp(ip: String): Option[Array[Byte]] = { + Try(InetAddress.getByName(ip).getAddress).toOption + } + } + + // The cached trie, atomically updated + private val cachedTrie = new AtomicReference[CidrTrie](new CidrTrie()) + private val lastRefreshTime = new AtomicReference[Long](0L) + private val isInitialized = new AtomicReference[Boolean](false) + + // Background refresh scheduler + private lazy val scheduler = { + val s = Executors.newSingleThreadScheduledExecutor((r: Runnable) => { + val t = new Thread(r, "gcp-ip-range-refresh") + t.setDaemon(true) + t + }) + s.scheduleAtFixedRate( + () => refresh(), + REFRESH_INTERVAL_HOURS, + REFRESH_INTERVAL_HOURS, + TimeUnit.HOURS + ) + s + } + + /** + * Look up the GCP region for a given IP address. + * + * @param ip The IP address to look up (IPv4 or IPv6) + * @return Optional GCP region (e.g., "us-central1", "europe-west1") + */ + def lookupRegion(ip: String): Option[String] = { + ensureInitialized() + cachedTrie.get().lookup(ip) + } + + /** + * Check if an IP is in any GCP range. + */ + def isGcpIp(ip: String): Boolean = { + lookupRegion(ip).isDefined + } + + /** + * Ensure the IP ranges are loaded at least once. + */ + def ensureInitialized(): Unit = { + if (!isInitialized.get()) { + synchronized { + if (!isInitialized.get()) { + refresh() + // Touch the scheduler to start background refreshes + scheduler + } + } + } + } + + /** + * Refresh the IP ranges from GCP's endpoint. + * This is called automatically on a schedule, but can also be called manually. + */ + def refresh(): Unit = { + try { + fetchAndParse() match { + case Success(ranges) => + val newTrie = buildTrie(ranges) + cachedTrie.set(newTrie) + lastRefreshTime.set(System.currentTimeMillis()) + isInitialized.set(true) + // scalastyle:off println + System.out.println(s"[GcpIpRangeLookup] Loaded ${ranges.prefixes.size} IP ranges " + + s"(syncToken: ${ranges.syncToken})") + // scalastyle:on println + case Failure(e) => + // scalastyle:off println + System.err.println(s"[GcpIpRangeLookup] Failed to refresh IP ranges: ${e.getMessage}") + // scalastyle:on println + // Keep using the old trie if we have one + if (!isInitialized.get()) { + // First-time failure - create empty trie + cachedTrie.set(new CidrTrie()) + isInitialized.set(true) + } + } + } catch { + case NonFatal(e) => + // scalastyle:off println + System.err.println(s"[GcpIpRangeLookup] Unexpected error during refresh: ${e.getMessage}") + // scalastyle:on println + } + } + + /** + * Fetch and parse the GCP IP ranges JSON. + */ + private def fetchAndParse(): Try[GcpIpRanges] = { + Try { + val url = new URL(GCP_IP_RANGES_URL) + val conn = url.openConnection().asInstanceOf[HttpURLConnection] + conn.setConnectTimeout(CONNECTION_TIMEOUT_MS) + conn.setReadTimeout(READ_TIMEOUT_MS) + conn.setRequestProperty("Accept", "application/json") + + try { + val responseCode = conn.getResponseCode + if (responseCode != 200) { + throw new RuntimeException(s"HTTP $responseCode from GCP IP ranges endpoint") + } + + val inputStream = conn.getInputStream + val content = scala.io.Source.fromInputStream(inputStream).mkString + inputStream.close() + + JsonUtils.fromJson[GcpIpRanges](content) + } finally { + conn.disconnect() + } + } + } + + /** + * Build a CIDR trie from the parsed IP ranges. + */ + private def buildTrie(ranges: GcpIpRanges): CidrTrie = { + val trie = new CidrTrie() + + for (prefix <- ranges.prefixes) { + // Only include "Google Cloud" service entries with a valid scope + // Skip "global" scope as it doesn't help with region detection + if (prefix.service == "Google Cloud" && prefix.scope.nonEmpty && + prefix.scope != "global") { + prefix.ipv4Prefix.foreach(cidr => trie.insert(cidr, prefix.scope)) + prefix.ipv6Prefix.foreach(cidr => trie.insert(cidr, prefix.scope)) + } + } + + trie + } + + /** + * Get the time of the last successful refresh, in milliseconds since epoch. + * Returns 0 if never refreshed. + */ + def getLastRefreshTime: Long = lastRefreshTime.get() + + /** + * For testing: load IP ranges from a pre-fetched JSON string. + */ + def loadFromJson(json: String): Unit = { + val ranges = JsonUtils.fromJson[GcpIpRanges](json) + val newTrie = buildTrie(ranges) + cachedTrie.set(newTrie) + lastRefreshTime.set(System.currentTimeMillis()) + isInitialized.set(true) + } + + /** + * For testing: reset to uninitialized state. + */ + def reset(): Unit = { + cachedTrie.set(new CidrTrie()) + lastRefreshTime.set(0L) + isInitialized.set(false) + } +} diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala index acf889a16..9b363a661 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -326,18 +326,21 @@ object GcpPricingTier { * * Detection priority: * 1. Private IP (10.x, 172.16-31.x, 192.168.x) => SAME_REGION (internal cluster) - * 2. Envoy metadata with GCP region extractable => SAME_REGION if matches sourceRegion, - * otherwise INTER_REGION - * 3. Envoy metadata present + GCP public IP => INTER_REGION (service mesh traffic) - * 4. X-Client-Region = "ZZ" => INTER_REGION (GCP internal) - * 5. Valid country code in X-Client-Region => INTERNET - * 6. Otherwise => UNKNOWN + * 2. GCP IP detected via IP range lookup => Use exact region from lookup + * a. If client region matches source region => SAME_REGION + * b. Otherwise => INTER_REGION (with exact client region for pricing) + * 3. Non-GCP IP with valid country code => INTERNET + * 4. Otherwise => UNKNOWN + * + * NOTE: The x-envoy-peer-metadata header is NOT used for region detection + * because it contains the INGRESS GATEWAY's location, not the actual + * client's location. We use GCP's published IP ranges instead. * * @param clientIp The client IP address (from X-Forwarded-For) - * @param envoyPeerMetadata The X-Envoy-Peer-Metadata header value + * @param envoyPeerMetadata Unused, kept for API compatibility * @param clientRegion The client region from X-Client-Region header * @param detectGcpTraffic Whether GCP traffic detection is enabled - * @param sourceRegion The GCP region where this server runs (for same-region detection) + * @param sourceRegion The GCP region where this server runs * @return Tuple of (EgressType, Optional destination GCP region) */ def determineEgressType( @@ -352,54 +355,45 @@ object GcpPricingTier { return (SAME_REGION, None) } - // If GCP traffic detection is enabled + // If GCP traffic detection is enabled, use IP range lookup for accurate region detection if (detectGcpTraffic) { - // Try to extract specific GCP region from Envoy metadata - val clientGcpRegion = envoyPeerMetadata.flatMap(extractGcpRegionFromEnvoyMetadata) - if (clientGcpRegion.isDefined) { - // If we know both source and client regions, check if they match - val isSameRegion = (sourceRegion, clientGcpRegion) match { - case (Some(src), Some(dst)) => normalizeRegion(src) == normalizeRegion(dst) - case _ => false - } - if (isSameRegion) { - return (SAME_REGION, clientGcpRegion) - } - return (INTER_REGION, clientGcpRegion) - } - - // If Envoy metadata is present (service mesh) AND client IP is GCP public range, - // this is inter-region GCP traffic even if we can't extract the specific region - val hasEnvoyMetadata = envoyPeerMetadata.isDefined - val isGcpIp = clientIp.exists(isGcpPublicIp) - if (hasEnvoyMetadata && isGcpIp) { - return (INTER_REGION, None) - } - - // If client IP is GCP range but no Envoy metadata, still likely GCP traffic - // (e.g., Cloud Run, Cloud Functions calling directly) - if (isGcpIp) { - // Check if region header suggests internal traffic - clientRegion match { - // scalastyle:off caselocale - case Some(region) if region.toUpperCase == "ZZ" => - // scalastyle:on caselocale - return (INTER_REGION, None) - case _ => - // GCP IP but valid country code - could be GCP service with external IP - // Classify as inter-region since it's still GCP-to-GCP - return (INTER_REGION, None) - } + clientIp.flatMap(GcpIpRangeLookup.lookupRegion) match { + case Some(clientGcpRegion) => + // We have exact GCP region from IP lookup + val isSameRegion = sourceRegion match { + case Some(src) => normalizeRegion(src) == normalizeRegion(clientGcpRegion) + case None => false + } + if (isSameRegion) { + return (SAME_REGION, Some(clientGcpRegion)) + } + return (INTER_REGION, Some(clientGcpRegion)) + + case None => + // IP not found in GCP ranges - check if it might still be GCP + // (fallback for any gaps in the published ranges) + if (clientIp.exists(isGcpPublicIp)) { + // IP looks like GCP but not in ranges - classify as inter-region conservatively + clientRegion match { + // scalastyle:off caselocale + case Some(region) if region.toUpperCase == "ZZ" => + // scalastyle:on caselocale + return (INTER_REGION, None) + case _ => + return (INTER_REGION, None) + } + } + // Not a GCP IP - fall through to internet traffic detection } } - // Check X-Client-Region for remaining cases + // Non-GCP client IP - this is internet traffic clientRegion match { // scalastyle:off caselocale case Some(region) if region.toUpperCase == "ZZ" => // scalastyle:on caselocale - // ZZ typically means GCP couldn't determine the location (internal traffic) - (INTER_REGION, None) + // ZZ typically means GCP couldn't determine the location + (INTERNET, None) case Some(_) => // Has a valid country code - this is internet traffic (INTERNET, None) diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookupSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookupSuite.scala new file mode 100644 index 000000000..08ac384b3 --- /dev/null +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookupSuite.scala @@ -0,0 +1,246 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.sharing.server.telemetry + +import org.scalatest.{BeforeAndAfterEach, FunSuite} + +class GcpIpRangeLookupSuite extends FunSuite with BeforeAndAfterEach { + + // Sample GCP IP ranges JSON for testing + private val testRangesJson = + """ + { + "syncToken": "1234567890", + "creationTime": "2026-06-01T00:00:00.000000", + "prefixes": [ + { + "ipv4Prefix": "34.44.0.0/15", + "service": "Google Cloud", + "scope": "us-central1" + }, + { + "ipv4Prefix": "34.72.0.0/16", + "service": "Google Cloud", + "scope": "us-central1" + }, + { + "ipv4Prefix": "34.73.0.0/16", + "service": "Google Cloud", + "scope": "us-east1" + }, + { + "ipv4Prefix": "35.187.0.0/17", + "service": "Google Cloud", + "scope": "europe-west1" + }, + { + "ipv4Prefix": "34.87.0.0/17", + "service": "Google Cloud", + "scope": "asia-southeast1" + }, + { + "ipv4Prefix": "34.151.64.0/18", + "service": "Google Cloud", + "scope": "australia-southeast1" + }, + { + "ipv4Prefix": "34.95.128.0/17", + "service": "Google Cloud", + "scope": "southamerica-east1" + }, + { + "ipv4Prefix": "34.8.0.0/16", + "service": "Google Cloud", + "scope": "global" + }, + { + "ipv6Prefix": "2600:1900:4000::/44", + "service": "Google Cloud", + "scope": "us-central1" + } + ] + } + """ + + override def beforeEach(): Unit = { + GcpIpRangeLookup.reset() + } + + test("lookupRegion returns None for empty/null IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + assert(GcpIpRangeLookup.lookupRegion(null).isEmpty) + assert(GcpIpRangeLookup.lookupRegion("").isEmpty) + } + + test("lookupRegion returns None for non-GCP IPs") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + assert(GcpIpRangeLookup.lookupRegion("8.8.8.8").isEmpty) + assert(GcpIpRangeLookup.lookupRegion("1.2.3.4").isEmpty) + assert(GcpIpRangeLookup.lookupRegion("217.9.6.27").isEmpty) // Malta IP + } + + test("lookupRegion returns correct region for us-central1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.44.0.0/15 covers 34.44.0.0 - 34.45.255.255 + assert(GcpIpRangeLookup.lookupRegion("34.44.0.1") == Some("us-central1")) + assert(GcpIpRangeLookup.lookupRegion("34.45.22.184") == Some("us-central1")) + assert(GcpIpRangeLookup.lookupRegion("34.45.255.255") == Some("us-central1")) + + // 34.72.0.0/16 + assert(GcpIpRangeLookup.lookupRegion("34.72.100.50") == Some("us-central1")) + } + + test("lookupRegion returns correct region for us-east1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.73.0.0/16 + assert(GcpIpRangeLookup.lookupRegion("34.73.0.1") == Some("us-east1")) + assert(GcpIpRangeLookup.lookupRegion("34.73.128.100") == Some("us-east1")) + } + + test("lookupRegion returns correct region for europe-west1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 35.187.0.0/17 covers 35.187.0.0 - 35.187.127.255 + assert(GcpIpRangeLookup.lookupRegion("35.187.0.1") == Some("europe-west1")) + assert(GcpIpRangeLookup.lookupRegion("35.187.64.100") == Some("europe-west1")) + assert(GcpIpRangeLookup.lookupRegion("35.187.127.255") == Some("europe-west1")) + + // 35.187.128.0 is outside the /17 + assert(GcpIpRangeLookup.lookupRegion("35.187.128.0").isEmpty) + } + + test("lookupRegion returns correct region for asia-southeast1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.87.0.0/17 + assert(GcpIpRangeLookup.lookupRegion("34.87.0.1") == Some("asia-southeast1")) + assert(GcpIpRangeLookup.lookupRegion("34.87.100.50") == Some("asia-southeast1")) + } + + test("lookupRegion returns correct region for australia-southeast1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.151.64.0/18 covers 34.151.64.0 - 34.151.127.255 + assert(GcpIpRangeLookup.lookupRegion("34.151.64.1") == Some("australia-southeast1")) + assert(GcpIpRangeLookup.lookupRegion("34.151.100.50") == Some("australia-southeast1")) + } + + test("lookupRegion returns correct region for southamerica-east1 IP") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.95.128.0/17 + assert(GcpIpRangeLookup.lookupRegion("34.95.128.1") == Some("southamerica-east1")) + assert(GcpIpRangeLookup.lookupRegion("34.95.200.50") == Some("southamerica-east1")) + } + + test("lookupRegion skips global scope ranges") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 34.8.0.0/16 is in the JSON but with scope "global" - should be skipped + assert(GcpIpRangeLookup.lookupRegion("34.8.1.1").isEmpty) + } + + test("isGcpIp returns true for GCP IPs") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + assert(GcpIpRangeLookup.isGcpIp("34.45.22.184")) + assert(GcpIpRangeLookup.isGcpIp("34.73.100.50")) + assert(GcpIpRangeLookup.isGcpIp("35.187.64.100")) + } + + test("isGcpIp returns false for non-GCP IPs") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + assert(!GcpIpRangeLookup.isGcpIp("8.8.8.8")) + assert(!GcpIpRangeLookup.isGcpIp("1.2.3.4")) + assert(!GcpIpRangeLookup.isGcpIp("192.168.1.1")) + } + + test("handles invalid IP addresses gracefully") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + assert(GcpIpRangeLookup.lookupRegion("not-an-ip").isEmpty) + assert(GcpIpRangeLookup.lookupRegion("256.256.256.256").isEmpty) + assert(GcpIpRangeLookup.lookupRegion("34.45").isEmpty) + } + + test("handles IPv6 addresses") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + + // 2600:1900:4000::/44 is us-central1 + assert(GcpIpRangeLookup.lookupRegion("2600:1900:4000::1") == Some("us-central1")) + assert(GcpIpRangeLookup.lookupRegion("2600:1900:4001:1234::5678") == Some("us-central1")) + + // Outside the range + assert(GcpIpRangeLookup.lookupRegion("2600:1901::1").isEmpty) + } + + test("longest prefix match selects most specific range") { + // Create test data with overlapping ranges + val overlappingRangesJson = + """ + { + "syncToken": "123", + "creationTime": "2026-06-01T00:00:00.000000", + "prefixes": [ + { + "ipv4Prefix": "34.0.0.0/8", + "service": "Google Cloud", + "scope": "us-west1" + }, + { + "ipv4Prefix": "34.45.0.0/16", + "service": "Google Cloud", + "scope": "us-central1" + }, + { + "ipv4Prefix": "34.45.22.0/24", + "service": "Google Cloud", + "scope": "us-east1" + } + ] + } + """ + GcpIpRangeLookup.loadFromJson(overlappingRangesJson) + + // Most specific match wins + assert(GcpIpRangeLookup.lookupRegion("34.45.22.184") == Some("us-east1")) + assert(GcpIpRangeLookup.lookupRegion("34.45.100.1") == Some("us-central1")) + assert(GcpIpRangeLookup.lookupRegion("34.100.1.1") == Some("us-west1")) + } + + test("returns None for IPs not in loaded ranges") { + // After loading test ranges, IPs not in those ranges should return None + GcpIpRangeLookup.loadFromJson(testRangesJson) + // This IP is not in our test ranges + assert(GcpIpRangeLookup.lookupRegion("34.200.200.200").isEmpty) + } + + test("getLastRefreshTime updates after loadFromJson") { + val timeBefore = GcpIpRangeLookup.getLastRefreshTime + Thread.sleep(10) // Small delay to ensure time difference + GcpIpRangeLookup.loadFromJson(testRangesJson) + assert(GcpIpRangeLookup.getLastRefreshTime > timeBefore) + } + + test("getLastRefreshTime returns non-zero after load") { + GcpIpRangeLookup.loadFromJson(testRangesJson) + assert(GcpIpRangeLookup.getLastRefreshTime > 0) + } +} diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala index d76918c50..42d2eb805 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala @@ -18,13 +18,73 @@ package io.delta.sharing.server.telemetry import java.util.Base64 -import org.scalatest.FunSuite +import org.scalatest.{BeforeAndAfterEach, FunSuite} -class GcpPricingTierSuite extends FunSuite { +class GcpPricingTierSuite extends FunSuite with BeforeAndAfterEach { import GcpPricingTier._ import GcpPricingTier.Continent._ import GcpPricingTier.EgressType._ + // Sample GCP IP ranges for testing + private val testRangesJson = + """ + { + "syncToken": "1234567890", + "creationTime": "2026-06-01T00:00:00.000000", + "prefixes": [ + { + "ipv4Prefix": "34.44.0.0/15", + "service": "Google Cloud", + "scope": "us-central1" + }, + { + "ipv4Prefix": "34.72.0.0/16", + "service": "Google Cloud", + "scope": "us-central1" + }, + { + "ipv4Prefix": "34.73.0.0/16", + "service": "Google Cloud", + "scope": "us-east1" + }, + { + "ipv4Prefix": "35.187.0.0/17", + "service": "Google Cloud", + "scope": "europe-west1" + }, + { + "ipv4Prefix": "34.87.0.0/17", + "service": "Google Cloud", + "scope": "asia-southeast1" + }, + { + "ipv4Prefix": "34.151.64.0/18", + "service": "Google Cloud", + "scope": "australia-southeast1" + }, + { + "ipv4Prefix": "34.95.128.0/17", + "service": "Google Cloud", + "scope": "southamerica-east1" + }, + { + "ipv4Prefix": "34.100.0.0/16", + "service": "Google Cloud", + "scope": "us-central1" + } + ] + } + """ + + override def beforeEach(): Unit = { + GcpIpRangeLookup.reset() + GcpIpRangeLookup.loadFromJson(testRangesJson) + } + + override def afterEach(): Unit = { + GcpIpRangeLookup.reset() + } + // ===== Continent Mapping Tests ===== test("continentFromGcpRegion maps US regions to NA") { @@ -231,9 +291,10 @@ class GcpPricingTierSuite extends FunSuite { assert(destRegion.isEmpty) } - test("determineEgressType returns INTER_REGION when ZZ region detected") { + test("determineEgressType returns INTER_REGION for GCP IP not in ranges with ZZ region") { + // Use a GCP-looking IP that's not in our test ranges val (egressType, destRegion) = determineEgressType( - clientIp = Some("34.100.0.1"), + clientIp = Some("34.200.0.1"), // GCP-looking IP not in test ranges envoyPeerMetadata = None, clientRegion = Some("ZZ"), detectGcpTraffic = true) @@ -251,14 +312,27 @@ class GcpPricingTierSuite extends FunSuite { assert(destRegion.isEmpty) } - test("determineEgressType returns SAME_REGION when client GCP region matches source region") { - // Create Envoy metadata with gcp_location + test("determineEgressType returns INTERNET for non-GCP IP even with Envoy metadata") { + // This is the critical test: non-GCP IP (like from Malta) should be INTERNET + // even if Envoy metadata contains gcp_location (which is the ingress gateway's location) val payload = """{"gcp_location":"us-central1-f"}""" val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) val (egressType, destRegion) = determineEgressType( - clientIp = Some("34.100.0.1"), - envoyPeerMetadata = Some(encoded), + clientIp = Some("217.9.6.27"), // Non-GCP IP from Malta (not in any GCP range) + envoyPeerMetadata = Some(encoded), // Ingress gateway's metadata (should be ignored) + clientRegion = Some("MT"), + detectGcpTraffic = true, + sourceRegion = Some("us-central1")) + assert(egressType == INTERNET, "Non-GCP IP should be classified as INTERNET") + assert(destRegion.isEmpty, "No GCP region for non-GCP IPs") + } + + test("determineEgressType returns SAME_REGION when client GCP region matches source region") { + // Use IP from us-central1 range (34.100.0.0/16 is in our test data) + val (egressType, destRegion) = determineEgressType( + clientIp = Some("34.100.0.1"), // GCP IP in us-central1 range + envoyPeerMetadata = None, // Not used anymore clientRegion = Some("US"), detectGcpTraffic = true, sourceRegion = Some("us-central1")) @@ -267,13 +341,10 @@ class GcpPricingTierSuite extends FunSuite { } test("determineEgressType returns INTER_REGION when client GCP region differs from source region") { - // Create Envoy metadata with gcp_location in different region - val payload = """{"gcp_location":"europe-west1-b"}""" - val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) - + // Use IP from europe-west1 range (35.187.0.0/17 is in our test data) val (egressType, destRegion) = determineEgressType( - clientIp = Some("34.100.0.1"), - envoyPeerMetadata = Some(encoded), + clientIp = Some("35.187.64.100"), // GCP IP in europe-west1 range + envoyPeerMetadata = None, // Not used anymore clientRegion = Some("DE"), detectGcpTraffic = true, sourceRegion = Some("us-central1")) @@ -282,20 +353,34 @@ class GcpPricingTierSuite extends FunSuite { } test("determineEgressType returns SAME_REGION with zone suffix variations") { - // Test that "us-central1" matches "us-central1-f" - val payload = """{"gcp_location":"us-central1-a"}""" - val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) - + // Test that "us-central1" source matches client IP in us-central1 range val (egressType, destRegion) = determineEgressType( - clientIp = Some("34.100.0.1"), - envoyPeerMetadata = Some(encoded), + clientIp = Some("34.100.0.1"), // GCP IP in us-central1 range + envoyPeerMetadata = None, // Not used anymore clientRegion = Some("US"), detectGcpTraffic = true, - sourceRegion = Some("us-central1-f")) + sourceRegion = Some("us-central1-f")) // With zone suffix assert(egressType == SAME_REGION) assert(destRegion.contains("us-central1")) } + test("determineEgressType uses IP lookup instead of Envoy metadata") { + // Even with Envoy metadata pointing to us-central1, if the IP is in europe-west1, + // the lookup should return europe-west1 + val payload = """{"gcp_location":"us-central1-f"}""" + val encoded = Base64.getEncoder.encodeToString(payload.getBytes("UTF-8")) + + val (egressType, destRegion) = determineEgressType( + clientIp = Some("35.187.64.100"), // GCP IP in europe-west1 range + envoyPeerMetadata = Some(encoded), // Should be ignored + clientRegion = Some("DE"), + detectGcpTraffic = true, + sourceRegion = Some("us-central1")) + assert(egressType == INTER_REGION) + // The region should come from IP lookup, not Envoy metadata + assert(destRegion.contains("europe-west1")) + } + // ===== Envoy Metadata Parsing Tests ===== test("extractGcpRegionFromEnvoyMetadata returns None for null/empty") { From 95f641750dfac00d5990e0f319211230f7910e9d Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 8 Jun 2026 13:30:04 +0300 Subject: [PATCH 12/15] Update docs --- docs/PER_SHARE_EGRESS_MONITORING.md | 179 +++++++--------------------- 1 file changed, 44 insertions(+), 135 deletions(-) diff --git a/docs/PER_SHARE_EGRESS_MONITORING.md b/docs/PER_SHARE_EGRESS_MONITORING.md index 4f7aaebef..438b6dedd 100644 --- a/docs/PER_SHARE_EGRESS_MONITORING.md +++ b/docs/PER_SHARE_EGRESS_MONITORING.md @@ -1,27 +1,18 @@ -# Per-Share Egress Monitoring with Pricing Group Determination +# Per-Share Egress Monitoring ## Overview -This document describes the new per-share egress monitoring feature introduced in the ZING-43690 branch. The feature provides: +Structured access logs for egress tracking with automatic GCP pricing tier classification. -1. **Share-attributed egress tracking** — Structured access logs for every query and CDF request -2. **GCP pricing tier classification** — Automatic determination of egress cost categories -3. **Traffic type detection** — Distinguishing between internet, inter-region GCP, and same-region traffic - -## Feature Summary - -When enabled, the Delta Sharing Server emits structured JSON log entries for each data access request. These logs include: - -- **Share identification**: share name, schema, table -- **Data volume**: total egress bytes transferred -- **Pricing tier**: GCP egress pricing category (e.g., `internet_to_na_eu`, `interregion_na_to_na`) -- **Client context**: region code, request type +When enabled, each data access emits JSON logs with: +- Share/schema/table identification +- Total egress bytes +- Pricing tier (e.g., `internet_to_na_eu`, `interregion_na_to_eu`) +- Client region code --- -## Pricing Group Types - -The system classifies egress traffic into pricing tiers based on GCP's network pricing structure. There are three main categories: +## Pricing Tiers ### 1. Free/Internal Traffic @@ -59,19 +50,14 @@ Traffic leaving GCP to external clients: | Tier | Description | |------|-------------| -| `interregion_unknown` | Inter-region traffic detected but source region not configured | | `unknown` | Unable to determine pricing tier | --- -## How Pricing Groups Are Resolved - -The pricing tier is determined through a multi-step detection process: +## How Pricing Tiers Are Resolved ### Step 1: Determine Egress Type -The system first classifies the traffic type based on available signals: - ``` ┌─────────────────────────────────────────────────────────────────┐ │ Egress Type Detection │ @@ -79,29 +65,26 @@ The system first classifies the traffic type based on available signals: │ 1. Client IP is private (10.x, 172.16-31.x, 192.168.x)? │ │ → SAME_REGION (internal cluster traffic) │ │ │ -│ 2. GCP detection enabled AND Envoy metadata contains region? │ -│ a. Extracted client region matches configured sourceRegion? │ -│ → SAME_REGION (same-region GCP traffic) │ -│ b. Otherwise │ -│ → INTER_REGION (with specific GCP region) │ -│ │ -│ 3. Envoy metadata present AND client IP is GCP range (34.x/35.x)?│ -│ → INTER_REGION (service mesh traffic) │ -│ │ -│ 4. Client IP is GCP range without Envoy metadata? │ -│ → INTER_REGION (GCP service calling directly) │ +│ 2. GCP IP Range Lookup (from cloud.json) finds client IP? │ +│ a. Client GCP region matches sourceRegion? │ +│ → SAME_REGION │ +│ b. Client GCP region differs from sourceRegion? │ +│ → INTER_REGION (with exact destination region) │ │ │ -│ 5. X-Client-Region header = "ZZ"? │ -│ → INTER_REGION (GCP internal, location unknown) │ +│ 3. Client IP looks like GCP (34.x/35.x) but not in ranges? │ +│ → INTER_REGION (conservative fallback) │ │ │ -│ 6. X-Client-Region contains valid country code? │ -│ → INTERNET (external client) │ +│ 4. Non-GCP IP with valid country code? │ +│ → INTERNET │ │ │ -│ 7. No region header present? │ +│ 5. No region header? │ │ → UNKNOWN │ └─────────────────────────────────────────────────────────────────┘ ``` +The system fetches GCP's published IP ranges from `https://www.gstatic.com/ipranges/cloud.json` +and uses a CIDR trie for efficient IP-to-region lookup (refreshed every 24 hours). + ### Step 2: Determine Continents Based on the source region (server configuration) and destination (detected from headers): @@ -135,36 +118,20 @@ Based on egress type and continent pair: ## Configuration -Enable access logging in your server configuration: - ```yaml accessLogging: - # Enable/disable the feature enabled: true - - # GCP region where this server runs (required for inter-region pricing) - sourceRegion: "us-central1" - - # Enable GCP inter-region traffic detection - detectGcpTraffic: true - - # Header containing client country code (set via load balancer) - clientRegionHeader: "x-client-region" - - # Header containing client IP chain - clientIpHeader: "x-forwarded-for" - - # Optional: custom pricing group mappings - pricingGroups: {} - - # Fallback when no location can be resolved - defaultPricingGroup: "unknown" + sourceRegion: "us-central1" # GCP region where server runs + detectGcpTraffic: true # Enable GCP IP range lookup + clientRegionHeader: "x-client-region" # Header with country code + clientRegionSubdivisionHeader: "x-client-region-subdivision" + clientIpHeader: "x-forwarded-for" # Header with client IP chain + defaultPricingGroup: "unknown" # Fallback when location unknown ``` -### GCP Load Balancer Configuration - -To populate the `X-Client-Region` header, configure your GCP load balancer backend with custom request headers: +### GCP Load Balancer Headers +Configure custom request headers on your backend: ``` X-Client-Region: {client_region} X-Client-Region-Subdivision: {client_region_subdivision} @@ -174,15 +141,12 @@ X-Client-Region-Subdivision: {client_region_subdivision} ## Log Output -### ACCESS_LOG Entry - -Emitted for each query/CDF request with non-zero egress: - +**ACCESS_LOG** — Emitted for each request with non-zero egress: ```json { "logType": "ACCESS_LOG", "share": "myshare", - "schema": "myschema", + "schema": "myschema", "table": "mytable", "egressBytes": 1048576, "pricingTier": "internet_to_na_eu", @@ -192,60 +156,25 @@ Emitted for each query/CDF request with non-zero egress: } ``` -### PRICING_CONTEXT Entry - -Emitted alongside ACCESS_LOG for debugging and verification: - +**PRICING_CONTEXT** — Debug entry with detection details: ```json { "logType": "PRICING_CONTEXT", - "share": "myshare", - "table": "mytable", - "timestampMs": 1717502400000, "clientIp": "203.0.113.45", - "clientIpSource": "x-forwarded-for", - "rawRegionHeader": "US", - "regionHeaderSource": "x-client-region", - "hasEnvoyMetadata": false, "isGcpIp": false, "egressType": "internet", "sourceRegion": "us-central1", - "sourceContinent": "NA", "destinationContinent": "NA", "pricingTier": "internet_to_na_eu" } ``` -### REQUEST_HEADERS Entry - -Emitted alongside ACCESS_LOG and PRICING_CONTEXT for debugging header detection issues: - -```json -{ - "logType": "REQUEST_HEADERS", - "share": "myshare", - "table": "mytable", - "timestampMs": 1717502400000, - "headers": { - "x-forwarded-for": "34.45.22.184", - "x-client-region": "US", - "x-envoy-peer-metadata": "eyJnY3BfbG9jYXRpb24iOiJ1cy1jZW50cmFsMS1mIn0=", - "content-type": "application/json", - "authorization": "Bearer ***" - } -} -``` - -This log entry contains all request headers (lowercased) and is useful for debugging -why a particular pricing tier was assigned. It helps verify whether the expected -headers (like `x-envoy-peer-metadata` with `gcp_location`) are being passed correctly. +**REQUEST_HEADERS** — All headers for debugging (keys lowercased). --- ## Header Detection Priority -The system checks multiple headers in priority order: - **Region Headers:** 1. Configured `clientRegionHeader` (default: `x-client-region`) 2. `x-appengine-country` @@ -258,47 +187,27 @@ The system checks multiple headers in priority order: 3. `x-real-ip` 4. `true-client-ip` -For IP chains (comma-separated), the first public IP is selected. - --- -## Implementation Details - -### Key Components +## Implementation | File | Purpose | |------|---------| -| `GcpPricingTier.scala` | Continent mapping, egress type detection, pricing tier calculation | +| `GcpPricingTier.scala` | Continent mapping, egress type detection, pricing calculation | +| `GcpIpRangeLookup.scala` | GCP IP range fetching and CIDR trie lookup | | `AccessLogEmitter.scala` | Log entry models and JSON emission | -| `DeltaSharingService.scala` | Integration points for query and CDF endpoints | -| `ServerConfig.scala` | `AccessLoggingConfig` configuration model | +| `DeltaSharingService.scala` | Integration for query/CDF endpoints | +| `ServerConfig.scala` | `AccessLoggingConfig` model | ### Egress Bytes Calculation -Egress bytes are calculated by summing the `size` field from all file actions in the response: -- `AddFile` — Regular query results -- `AddFileForCDF` — CDF add file actions -- `AddCDCFile` — CDF change data capture files - ---- - -## Deployment Configurations - -The feature is enabled in production environments with the following settings: - -**zcloud-prod (us-central1):** -```yaml -accessLogging: - enabled: true - sourceRegion: "us-central1" - detectGcpTraffic: true -``` +Sum of `size` from all file actions: `AddFile`, `AddFileForCDF`, `AddCDCFile`. --- ## Notes -- Logs are emitted to a dedicated logger (`delta.sharing.access`) for easy filtering in Cloud Logging -- Zero-byte requests are not logged -- The `PRICING_CONTEXT` log provides full audit trail for pricing decisions -- Pricing tier strings match GCP documentation for easier cost analysis +- Logs emitted to `delta.sharing.access` logger +- Zero-byte requests not logged +- GCP IP ranges refreshed every 24 hours +- Pricing tiers match GCP documentation From 23b6df69104058148365f1ee478526f79fdacbb5 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 8 Jun 2026 16:34:27 +0300 Subject: [PATCH 13/15] Cleanup --- docs/PER_SHARE_EGRESS_MONITORING.md | 4 ---- .../sharing/server/DeltaSharingService.scala | 7 +------ .../sharing/server/config/ServerConfig.scala | 15 +++------------ .../sharing/server/config/ServerConfigSuite.scala | 9 --------- .../server/telemetry/GcpPricingTierSuite.scala | 2 ++ .../conf/delta-sharing-server.yaml.template | 7 +------ 6 files changed, 7 insertions(+), 37 deletions(-) diff --git a/docs/PER_SHARE_EGRESS_MONITORING.md b/docs/PER_SHARE_EGRESS_MONITORING.md index 438b6dedd..e5dfa897d 100644 --- a/docs/PER_SHARE_EGRESS_MONITORING.md +++ b/docs/PER_SHARE_EGRESS_MONITORING.md @@ -124,9 +124,7 @@ accessLogging: sourceRegion: "us-central1" # GCP region where server runs detectGcpTraffic: true # Enable GCP IP range lookup clientRegionHeader: "x-client-region" # Header with country code - clientRegionSubdivisionHeader: "x-client-region-subdivision" clientIpHeader: "x-forwarded-for" # Header with client IP chain - defaultPricingGroup: "unknown" # Fallback when location unknown ``` ### GCP Load Balancer Headers @@ -184,8 +182,6 @@ X-Client-Region-Subdivision: {client_region_subdivision} **Client IP Headers:** 1. Configured `clientIpHeader` (default: `x-forwarded-for`) 2. `x-envoy-external-address` -3. `x-real-ip` -4. `true-client-ip` --- diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 0c8d5552c..896ac3fd0 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -888,14 +888,9 @@ object DeltaSharingService { "x-appengine-country", "cf-ipcountry", "cloudfront-viewer-country") - private val DefaultSubdivisionHeaders = Seq( - "x-client-region-subdivision", - "x-appengine-region") private val DefaultIpHeaders = Seq( "x-forwarded-for", - "x-envoy-external-address", - "x-real-ip", - "true-client-ip") + "x-envoy-external-address") private val DefaultPricingGroupsByRegion = Map( "US" -> "na_eu", "CA" -> "na_eu", diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index 4802f6706..b90ccd40f 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -139,16 +139,12 @@ case class AccessLoggingConfig( @BeanProperty var enabled: Boolean, // Header that contains the client region code (for example: US, DE). @BeanProperty var clientRegionHeader: String, - // Header that contains the client region subdivision (for example: USCA, DEBE). - @BeanProperty var clientRegionSubdivisionHeader: String, // Header that contains client IP or forwarding chain (for example: X-Forwarded-For). @BeanProperty var clientIpHeader: String, - // Optional mapping from region/subdivision codes to pricing group labels. - // Keys should be uppercase location codes (for example: US, DE, USCA). + // Optional mapping from region codes to pricing group labels. + // Keys should be uppercase location codes (for example: US, DE). // A wildcard key "*" can be used as a catch-all default. @BeanProperty var pricingGroups: java.util.Map[String, String], - // Fallback group when no mapping and no region are available. - @BeanProperty var defaultPricingGroup: String, // The GCP region where this server runs (for example: us-central1). // Used for pricing tier calculation based on source→destination pairs. @BeanProperty var sourceRegion: String, @@ -161,19 +157,14 @@ case class AccessLoggingConfig( this( enabled = false, clientRegionHeader = "x-client-region", - clientRegionSubdivisionHeader = "x-client-region-subdivision", clientIpHeader = "x-forwarded-for", pricingGroups = Collections.emptyMap(), - defaultPricingGroup = "unknown", sourceRegion = "", detectGcpTraffic = true) } override def checkConfig(): Unit = { - if (defaultPricingGroup == null || defaultPricingGroup.trim.isEmpty) { - throw new IllegalArgumentException( - "'defaultPricingGroup' in 'accessLogging' must be provided") - } + // No required fields to validate } } diff --git a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala index 2258cd0e6..0be6be3c9 100644 --- a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala @@ -125,10 +125,8 @@ class ServerConfigSuite extends FunSuite { val accessLogging = new AccessLoggingConfig() accessLogging.setEnabled(true) accessLogging.setClientRegionHeader("x-client-region") - accessLogging.setClientRegionSubdivisionHeader("x-client-region-subdivision") accessLogging.setClientIpHeader("x-forwarded-for") accessLogging.setPricingGroups(Collections.singletonMap("US", "na-tier-1")) - accessLogging.setDefaultPricingGroup("unknown") serverConfig.setAccessLogging(accessLogging) testConfig( """version: 1 @@ -142,11 +140,9 @@ class ServerConfigSuite extends FunSuite { |accessLogging: | enabled: true | clientRegionHeader: x-client-region - | clientRegionSubdivisionHeader: x-client-region-subdivision | clientIpHeader: x-forwarded-for | pricingGroups: | US: na-tier-1 - | defaultPricingGroup: unknown |""".stripMargin, serverConfig) } @@ -209,11 +205,6 @@ class ServerConfigSuite extends FunSuite { val accessLogging = new AccessLoggingConfig() accessLogging.setEnabled(true) accessLogging.checkConfig() - - accessLogging.setDefaultPricingGroup(" ") - assertInvalidConfig("'defaultPricingGroup' in 'accessLogging' must be provided") { - accessLogging.checkConfig() - } } test("SSLConfig") { diff --git a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala index 42d2eb805..3317951d7 100644 --- a/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/telemetry/GcpPricingTierSuite.scala @@ -340,7 +340,9 @@ class GcpPricingTierSuite extends FunSuite with BeforeAndAfterEach { assert(destRegion.contains("us-central1")) } + // scalastyle:off test("determineEgressType returns INTER_REGION when client GCP region differs from source region") { + // scalastyle:on // Use IP from europe-west1 range (35.187.0.0/17 is in our test data) val (egressType, destRegion) = determineEgressType( clientIp = Some("35.187.64.100"), // GCP IP in europe-west1 range diff --git a/server/src/universal/conf/delta-sharing-server.yaml.template b/server/src/universal/conf/delta-sharing-server.yaml.template index 40b0b565b..277fb5c37 100644 --- a/server/src/universal/conf/delta-sharing-server.yaml.template +++ b/server/src/universal/conf/delta-sharing-server.yaml.template @@ -74,18 +74,13 @@ accessLogging: # In GCP, configure your load balancer backend custom request headers to inject this. # Example: X-Client-Region:{client_region} clientRegionHeader: "x-client-region" - # Name of the incoming header that carries client subdivision (for example: USCA). - # Example: X-Client-Region-Subdivision:{client_region_subdivision} - clientRegionSubdivisionHeader: "x-client-region-subdivision" # Name of the incoming header that carries client IP information. # This is typically a forwarding chain header and does not require LB changes when already present. clientIpHeader: "x-forwarded-for" - # Optional mapping from region/subdivision codes to billing price groups. + # Optional mapping from region codes to billing price groups. # Use uppercase keys. A wildcard key "*" can be used as a catch-all. # If empty, server applies a built-in default map (na_eu/apac/latam buckets). pricingGroups: {} - # Fallback pricing group when no location can be resolved. - defaultPricingGroup: "unknown" # GCP region where this server runs (for example: us-central1). # Required for GCP pricing tier calculation. # Used to determine source→destination pairs for egress cost attribution. From e8adfca24ae279e3c4806cb8c810be6ac44ff4a8 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Mon, 8 Jun 2026 17:08:53 +0300 Subject: [PATCH 14/15] Address comments --- .github/workflows/build-and-test.yml | 2 +- README.md | 2 ++ .../sharing/server/DeltaSharingService.scala | 33 ++++++++++++++++--- .../sharing/server/config/ServerConfig.scala | 7 ++-- .../server/telemetry/AccessLogEmitter.scala | 26 ++++++++++++++- .../server/telemetry/GcpIpRangeLookup.scala | 17 ++++------ .../server/telemetry/GcpPricingTier.scala | 6 ++-- .../conf/delta-sharing-server.yaml.template | 9 +++-- 8 files changed, 75 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ba7b49e28..1b584f464 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -135,7 +135,7 @@ jobs: conda config --env --add pinned_packages python=$PYTHON_VERSION conda config --env --add pinned_packages pandas==$PANDAS_VERSION conda config --env --add pinned_packages pyarrow==$PYARROW_VERSION - conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION + conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION pip sed -i -e "/pandas/d" -e "/pyarrow/d" python/requirements-dev.txt conda install -c conda-forge --yes --file python/requirements-dev.txt conda list diff --git a/README.md b/README.md index 28f0f8c55..e324c23b8 100644 --- a/README.md +++ b/README.md @@ -447,7 +447,9 @@ Fields emitted in each log line: - `table` - `requestType` (`query` or `cdf_stream`) - `egressBytes` +- `pricingTier` - `timestampMs` +- `clientRegion` (if available) To enable this feature, configure the `accessLogging` block in the server yaml. diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 896ac3fd0..01bb426f3 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -213,6 +213,10 @@ class DeltaSharingService(serverConfig: ServerConfig) { schema: String, table: String, actions: Seq[Object]): Unit = { + if (!Option(serverConfig.getAccessLogging).exists(_.enabled)) { + return + } + val bytes = DeltaSharingService.extractEgressBytes(actions) if (bytes <= 0) { return @@ -273,6 +277,10 @@ class DeltaSharingService(serverConfig: ServerConfig) { schema: String, table: String, actions: Seq[Object]): Unit = { + if (!Option(serverConfig.getAccessLogging).exists(_.enabled)) { + return + } + val bytes = DeltaSharingService.extractEgressBytes(actions) if (bytes <= 0) { return @@ -1120,11 +1128,26 @@ object DeltaSharingService { } private def isValidIp(ip: String): Boolean = { - try { - InetAddress.getByName(ip) - true - } catch { - case _: Throwable => false + // Validate IP format without DNS resolution to avoid latency and security issues + val ipv4Pattern = """^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$""".r + val ipv6Pattern = """^([0-9a-fA-F:]+)$""".r + + ip match { + case ipv4Pattern(a, b, c, d) => + Seq(a, b, c, d).forall { octet => + val n = octet.toInt + n >= 0 && n <= 255 + } + case ipv6Pattern(_) => + try { + // For IPv6, use InetAddress but only for format validation + // This won't trigger DNS since it matches the IPv6 pattern + InetAddress.getByName(ip) + true + } catch { + case _: Throwable => false + } + case _ => false } } diff --git a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala index b90ccd40f..3cfd15df0 100644 --- a/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala +++ b/server/src/main/scala/io/delta/sharing/server/config/ServerConfig.scala @@ -144,13 +144,14 @@ case class AccessLoggingConfig( // Optional mapping from region codes to pricing group labels. // Keys should be uppercase location codes (for example: US, DE). // A wildcard key "*" can be used as a catch-all default. + // NOTE: pricingGroups is reserved for future use and not currently applied. @BeanProperty var pricingGroups: java.util.Map[String, String], // The GCP region where this server runs (for example: us-central1). // Used for pricing tier calculation based on source→destination pairs. @BeanProperty var sourceRegion: String, - // Enable GCP inter-region traffic detection via X-Envoy-Peer-Metadata header. - // When true, traffic from other GCP regions is classified as inter-region (cheaper) - // rather than internet egress. + // Enable GCP traffic detection using GCP's published IP ranges (cloud.json). + // When true, client IPs belonging to other GCP regions can be classified as inter-region + // (cheaper) rather than internet egress. Set to false to disable this detection. @BeanProperty var detectGcpTraffic: Boolean) extends ConfigItem { def this() = { diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index 212dd7853..3b1f3fdd5 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -16,6 +16,8 @@ package io.delta.sharing.server.telemetry +import java.util.Locale + import org.slf4j.LoggerFactory import io.delta.sharing.server.common.JsonUtils @@ -230,13 +232,35 @@ class JsonAccessLogEmitter extends AccessLogEmitter { logger.info(JsonUtils.toJson(logPayload)) } + // Headers that should be redacted to prevent leaking secrets/PII + private val sensitiveHeaders = Set( + "authorization", + "cookie", + "set-cookie", + "x-api-key", + "x-auth-token", + "proxy-authorization", + "www-authenticate", + "x-csrf-token", + "x-xsrf-token" + ) + override def recordHeaders(entry: RequestHeadersLogEntry): Unit = { + // Redact sensitive headers to prevent leaking secrets/PII + val redactedHeaders = entry.headers.map { case (key, value) => + if (sensitiveHeaders.contains(key.toLowerCase(Locale.ROOT))) { + key -> "[REDACTED]" + } else { + key -> value + } + } + val logPayload = Map( "logType" -> "REQUEST_HEADERS", "share" -> entry.share, "table" -> entry.table, "timestampMs" -> entry.timestampMs, - "headers" -> entry.headers + "headers" -> redactedHeaders ) logger.info(JsonUtils.toJson(logPayload)) diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala index ebda803f9..c1bdcdfea 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala @@ -24,6 +24,8 @@ import scala.collection.mutable import scala.util.{Failure, Success, Try} import scala.util.control.NonFatal +import org.slf4j.LoggerFactory + import io.delta.sharing.server.common.JsonUtils /** @@ -37,6 +39,8 @@ import io.delta.sharing.server.common.JsonUtils */ object GcpIpRangeLookup { + private val logger = LoggerFactory.getLogger("delta.sharing.gcp.ip.lookup") + private val GCP_IP_RANGES_URL = "https://www.gstatic.com/ipranges/cloud.json" private val REFRESH_INTERVAL_HOURS = 24 private val CONNECTION_TIMEOUT_MS = 10000 @@ -230,14 +234,9 @@ object GcpIpRangeLookup { cachedTrie.set(newTrie) lastRefreshTime.set(System.currentTimeMillis()) isInitialized.set(true) - // scalastyle:off println - System.out.println(s"[GcpIpRangeLookup] Loaded ${ranges.prefixes.size} IP ranges " + - s"(syncToken: ${ranges.syncToken})") - // scalastyle:on println + logger.info(s"Loaded ${ranges.prefixes.size} IP ranges (syncToken: ${ranges.syncToken})") case Failure(e) => - // scalastyle:off println - System.err.println(s"[GcpIpRangeLookup] Failed to refresh IP ranges: ${e.getMessage}") - // scalastyle:on println + logger.warn(s"Failed to refresh IP ranges: ${e.getMessage}") // Keep using the old trie if we have one if (!isInitialized.get()) { // First-time failure - create empty trie @@ -247,9 +246,7 @@ object GcpIpRangeLookup { } } catch { case NonFatal(e) => - // scalastyle:off println - System.err.println(s"[GcpIpRangeLookup] Unexpected error during refresh: ${e.getMessage}") - // scalastyle:on println + logger.error(s"Unexpected error during refresh: ${e.getMessage}") } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala index 9b363a661..1fa1aada5 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpPricingTier.scala @@ -355,8 +355,10 @@ object GcpPricingTier { return (SAME_REGION, None) } - // If GCP traffic detection is enabled, use IP range lookup for accurate region detection - if (detectGcpTraffic) { + // If GCP traffic detection is enabled and the IP looks like GCP, use IP range lookup for + // accurate region detection. This avoids unnecessary initialization/refresh overhead for + // non-GCP internet clients. + if (detectGcpTraffic && clientIp.exists(isGcpPublicIp)) { clientIp.flatMap(GcpIpRangeLookup.lookupRegion) match { case Some(clientGcpRegion) => // We have exact GCP region from IP lookup diff --git a/server/src/universal/conf/delta-sharing-server.yaml.template b/server/src/universal/conf/delta-sharing-server.yaml.template index 277fb5c37..fffd0a76a 100644 --- a/server/src/universal/conf/delta-sharing-server.yaml.template +++ b/server/src/universal/conf/delta-sharing-server.yaml.template @@ -79,13 +79,12 @@ accessLogging: clientIpHeader: "x-forwarded-for" # Optional mapping from region codes to billing price groups. # Use uppercase keys. A wildcard key "*" can be used as a catch-all. - # If empty, server applies a built-in default map (na_eu/apac/latam buckets). + # NOTE: pricingGroups is reserved for future use and not currently applied. pricingGroups: {} # GCP region where this server runs (for example: us-central1). - # Required for GCP pricing tier calculation. # Used to determine source→destination pairs for egress cost attribution. sourceRegion: "" - # Enable GCP inter-region traffic detection via X-Envoy-Peer-Metadata header. - # When true, traffic from other GCP regions is classified as inter-region (cheaper) - # rather than internet egress. Set to false to disable detection. + # Enable GCP traffic detection using GCP's published IP ranges (cloud.json). + # When true, client IPs belonging to other GCP regions can be classified as inter-region + # (cheaper) rather than internet egress. Set to false to disable this detection. detectGcpTraffic: true From 66636305f7b3258bbeaf476c119da45bffb60c73 Mon Sep 17 00:00:00 2001 From: Andriy Chorny Date: Tue, 9 Jun 2026 11:59:55 +0300 Subject: [PATCH 15/15] Address comments --- manifests/base/deployment.yaml | 2 ++ .../server/telemetry/AccessLogEmitter.scala | 10 ++++++-- .../server/telemetry/GcpIpRangeLookup.scala | 15 +++++++++--- .../server/DeltaSharingServiceSuite.scala | 24 +++++++++++++++++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/manifests/base/deployment.yaml b/manifests/base/deployment.yaml index 591770f01..de0549c45 100644 --- a/manifests/base/deployment.yaml +++ b/manifests/base/deployment.yaml @@ -44,6 +44,7 @@ spec: configMapKeyRef: name: project-common key: PROJECT_ID + optional: true volumeMounts: - name: base-config mountPath: /base-config @@ -59,6 +60,7 @@ spec: configMapKeyRef: name: project-common key: PROJECT_ID + optional: true - name: ZC_API_PROXY_DL_SHARING_BEARER valueFrom: secretKeyRef: diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala index 3b1f3fdd5..c9ee3ecb5 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/AccessLogEmitter.scala @@ -229,7 +229,10 @@ class JsonAccessLogEmitter extends AccessLogEmitter { val logPayload = basePayload ++ optionalPayload - logger.info(JsonUtils.toJson(logPayload)) + // Use DEBUG level for context logs to reduce volume/costs - only enable intentionally + if (logger.isDebugEnabled) { + logger.debug(JsonUtils.toJson(logPayload)) + } } // Headers that should be redacted to prevent leaking secrets/PII @@ -263,6 +266,9 @@ class JsonAccessLogEmitter extends AccessLogEmitter { "headers" -> redactedHeaders ) - logger.info(JsonUtils.toJson(logPayload)) + // Use DEBUG level for header logs - high-volume and can leak sensitive data + if (logger.isDebugEnabled) { + logger.debug(JsonUtils.toJson(logPayload)) + } } } diff --git a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala index c1bdcdfea..780870d59 100644 --- a/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala +++ b/server/src/main/scala/io/delta/sharing/server/telemetry/GcpIpRangeLookup.scala @@ -150,12 +150,18 @@ object GcpIpRangeLookup { /** * Parse a CIDR notation string into (ip bytes, prefix length). + * Validates that the prefix length is valid for the address type. */ private def parseCidr(cidr: String): Option[(Array[Byte], Int)] = { Try { val parts = cidr.split("/") val ipBytes = InetAddress.getByName(parts(0)).getAddress val prefixLen = parts(1).toInt + val maxPrefixLen = ipBytes.length * 8 // 32 for IPv4, 128 for IPv6 + if (prefixLen < 0 || prefixLen > maxPrefixLen) { + throw new IllegalArgumentException( + s"Invalid prefix length $prefixLen for ${ipBytes.length * 8}-bit address") + } (ipBytes, prefixLen) }.toOption } @@ -209,14 +215,17 @@ object GcpIpRangeLookup { /** * Ensure the IP ranges are loaded at least once. + * Initialization is async to avoid blocking the request path. */ def ensureInitialized(): Unit = { if (!isInitialized.get()) { synchronized { if (!isInitialized.get()) { - refresh() - // Touch the scheduler to start background refreshes - scheduler + // Mark as initialized immediately with empty trie to avoid blocking + // The background thread will populate it async + isInitialized.set(true) + // Start async refresh in background thread + scheduler.execute(() => refresh()) } } } diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index 991d95548..900e4aafa 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -38,10 +38,24 @@ import io.delta.sharing.server.common.actions.{ColumnMappingTableFeature, Deleti import io.delta.sharing.server.config.{AccessLoggingConfig, ServerConfig} import io.delta.sharing.server.model._ import io.delta.sharing.server.protocol._ +import io.delta.sharing.server.telemetry.GcpIpRangeLookup // scalastyle:off maxLineLength class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { + // Test IP ranges for deterministic GCP IP lookup tests + // Prevents live HTTP fetch to gstatic.com during tests + private val testGcpIpRangesJson = + """ + { + "syncToken": "test", + "creationTime": "2026-06-01T00:00:00.000000", + "prefixes": [ + {"ipv4Prefix": "34.100.0.0/16", "service": "Google Cloud", "scope": "us-central1"} + ] + } + """ + def shouldRunIntegrationTest: Boolean = { sys.env.get("AWS_ACCESS_KEY_ID").exists(_.length > 0) && sys.env.get("AZURE_TEST_ACCOUNT_KEY").exists(_.length > 0) && @@ -371,6 +385,9 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { } test("buildClientLocationContext calculates inter-region tier for ZZ region") { + // Load deterministic IP ranges to avoid live HTTP fetch + GcpIpRangeLookup.loadFromJson(testGcpIpRangesJson) + val cfg = new AccessLoggingConfig() cfg.setEnabled(true) cfg.setSourceRegion("us-central1") @@ -384,8 +401,8 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { cfg) assert(ctx.clientRegion.contains("ZZ")) - // Inter-region from NA (us-central1) to unknown -> internet fallback - assert(ctx.pricingTier.startsWith("inter") || ctx.pricingTier == "unknown") + // With deterministic IP ranges, 34.100.0.1 maps to us-central1 (same as sourceRegion) + assert(ctx.pricingTier == "same_region") } test("buildClientLocationContext calculates internet tier without source region") { @@ -406,6 +423,9 @@ class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { } test("buildClientLocationContext returns interregion_unknown for GCP IP without source") { + // Load deterministic IP ranges to avoid live HTTP fetch + GcpIpRangeLookup.loadFromJson(testGcpIpRangesJson) + val cfg = new AccessLoggingConfig() cfg.setEnabled(true) cfg.setDetectGcpTraffic(true)