Merge pull request #129 from ExpediaGroup/feature/gluesynccli_squashed

Added a gluesync CLI and 'cli' fat jar to run for on demand syncing.

Author: mroark1m

.editorconfig

@@ -0,0 +1,19 @@
+[*]
+indent_size=2
+indent_style=space
+charset=utf-8
+
+[*.java]
+continuation_indent_size=2
+trim_trailing_whitespace=true
+insert_final_newline=true
+max_line_length=120
+wildcard_import_limit=999
+ij_java_blank_lines_after_imports=1
+ij_java_blank_lines_before_imports=1
+ij_java_class_count_to_use_import_on_demand=999
+ij_java_names_count_to_use_import_on_demand=999
+ij_java_imports_layout=$java.**,|,$javax.**,|,$org.**,|,$*,$com.**,|,$com.expedia.**,|,$com.hotels.**,java.**,|,javax.**,|,org.**,|,*,|,com.**,|,com.expedia.**,|,com.hotels.**
+ij_java_keep_simple_methods_in_one_line=true
+ij_java_keep_simple_lambdas_in_one_line=true
+ij_java_keep_simple_classes_in_one_line = true

.gitignore

@@ -13,6 +13,9 @@
 *.iml
 *.iws
 
+# VSCode
+.vscode
+
 # Netbeans files
 nbactions.xml
 

CHANGELOG.md

@@ -3,6 +3,9 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 8.1.12 - 2025-09-15
+### Added
+- GlueSyncCli with fat jar for on-demand syncing.
 
 ## 8.1.11 - 2025-08-25
 ### Changed

apiary-metastore-events/kafka-metastore-events/kafka-metastore-listener/src/main/java/com/expediagroup/apiary/extensions/events/metastore/kafka/messaging/KafkaMessageSender.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2024 Expedia, Inc.
+ * Copyright (C) 2018-2025 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/privileges-grantor/privileges-grantor-lambda/pom.xml

@@ -1,5 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -16,7 +18,6 @@
     <aws-java-sdk-lambda.version>1.11.390</aws-java-sdk-lambda.version>
     <aws-lambda-java-core.version>1.2.0</aws-lambda-java-core.version>
     <aws-lambda-java-events.version>2.2.6</aws-lambda-java-events.version>
-    <system-rules.version>1.19.0</system-rules.version>
     <aws-serverless-java-container-core.version>1.3</aws-serverless-java-container-core.version>
   </properties>
 
@@ -66,7 +67,6 @@
     <dependency>
       <groupId>com.github.stefanbirkner</groupId>
       <artifactId>system-rules</artifactId>
-      <version>${system-rules.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -99,4 +99,4 @@
     </plugins>
   </build>
 
-</project>
+</project>

apiary-metastore-events/sns-metastore-events/apiary-metastore-listener/pom.xml

@@ -1,4 +1,6 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -56,7 +58,6 @@
     <dependency>
       <groupId>com.github.stefanbirkner</groupId>
       <artifactId>system-rules</artifactId>
-      <version>1.19.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -105,7 +106,8 @@
                   <shadedPattern>${shade.prefix}.com.fasterxml.jackson</shadedPattern>
                 </relocation>
                 <relocation>
-                  <!-- Note org.apache.httpcomponents groupId but org.apache.http package, this is relocated due
+                  <!-- Note org.apache.httpcomponents groupId but org.apache.http package, this is
+                  relocated due
                     to newer version used by com.amazonaws, older version is provided by /usr/lib/hive/ -->
                   <pattern>org.apache.http</pattern>
                   <shadedPattern>${shade.prefix}.org.apache.http</shadedPattern>
@@ -121,4 +123,4 @@
       </plugin>
     </plugins>
   </build>
-</project>
+</project>

hive-event-listeners/apiary-gluesync-listener/README.md

@@ -3,6 +3,8 @@
 ## Overview
 The GlueSync event listener is an optional Apiary component which, if enabled, will push metadata updates to an AWS Glue catalog.
 
+In addition to the GlueSync event listener, this component contains a CLI that uses the same code for on-demand syncing in case there are operational failures and you need to provide a quick fix. That is documented in [README_CLI.md](README_CLI.md). 
+
 ## Installation
 The listener can be activated by placing its jar file on the Hive metastore classpath and configuring Hive accordingly. For Apiary 
 this is done in [apiary-metastore-docker](https://github.com/ExpediaGroup/apiary-metastore-docker). 
@@ -24,4 +26,3 @@ If an archive is needed, this can be done per table by setting the Hive table pr
 This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
 
 Copyright 2018-2019 Expedia, Inc.
-

hive-event-listeners/apiary-gluesync-listener/README_CLI.md

@@ -0,0 +1,25 @@
+# Apiary GlueSync CLI
+
+## Overview
+The GlueSync CLI is a CLI that uses the same code as the hive event listener (with some additions) for on-demand syncing of a Hive metastore table to an AWS Glue database.
+
+One major addition for the GlueSync CLI is that it will always sync all partitions (if needed) for a given Hive table as opposed to the event driven sync which expects to receive individual events for each partition.
+
+## Installation
+The CLI gets packaged into a single fat jar with all dependencies so that you can run it simply by java -jar <PATHTOJAR>. The jar name is apiary-gluesync-listener-<version>-cli.jar.
+
+# Usage
+You need to provide two environment vars, AWS_REGION and THRIFT_CONNECTION_URI. Be careful not to point the THRIFT_URI to a waggledance endpoint or you may sync federated tables (unless you actually want that).
+
+Example:
+
+```
+AWS_REGION=us-east-1 THRIFT_CONNECTION_URI=thrift://my.hive.service:9083 java -jar ./target/apiary-gluesync-listener-8.1.12-cli.jar --database-name-regex 'lz_.*' --table-name-regex 'bookings.*' -v
+```
+
+For more usage details, pass the "-h" flag.
+
+# Legal
+This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
+
+Copyright 2018-2019 Expedia, Inc.