Skip to content

Commit b079095

Browse files
oscerdoscerd
authored
Add Databricks source and sink Kamelet (#2672)
Signed-off-by: Andrea Cosentino <ancosen@gmail.com>
1 parent e2ef32b commit b079095

4 files changed

Lines changed: 434 additions & 0 deletions

File tree

‎kamelets/databricks-sink.kamelet.yaml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# ---------------------------------------------------------------------------
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# ---------------------------------------------------------------------------
17+
apiVersion: camel.apache.org/v1
18+
kind: Kamelet
19+
metadata:
20+
name: databricks-sink
21+
annotations:
22+
camel.apache.org/kamelet.support.level: "Preview"
23+
camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
24+
camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMCAyNi45VjMzLjNMMCAyNS45VjE5LjVaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDE5LjVMMjAgMjYuOVYzMy4zTDQwIDI1LjlWMTkuNVoiLz4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMCAyNS45TDIwIDMzLjNWNDBMMCAzMi42VjI1LjlaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDI1LjlMMjAgMzMuM1Y0MEw0MCAzMi42VjI1LjlaIi8+Cjwvc3ZnPgo="
25+
camel.apache.org/provider: "Apache Software Foundation"
26+
camel.apache.org/kamelet.group: "SQL"
27+
camel.apache.org/kamelet.namespace: "Database"
28+
labels:
29+
camel.apache.org/kamelet.type: "sink"
30+
spec:
31+
definition:
32+
title: "Databricks Sink"
33+
description: Send data to a Databricks Database. This Kamelet expects a JSON-formatted body. Use key:value pairs to map the JSON fields and parameters. For Unity Catalog workspaces, specify catalog and schema parameters.
34+
required:
35+
- serverHostname
36+
- httpPath
37+
- accessToken
38+
- query
39+
type: object
40+
properties:
41+
serverHostname:
42+
title: Server Hostname
43+
description: The Databricks server hostname.
44+
type: string
45+
example: adb-1234567890123456.7.azuredatabricks.net
46+
serverPort:
47+
title: Server Port
48+
description: The server port for the Databricks data source.
49+
type: string
50+
default: 443
51+
httpPath:
52+
title: HTTP Path
53+
description: The HTTP path to the Databricks SQL Warehouse or cluster.
54+
type: string
55+
example: /sql/1.0/warehouses/abc123def456
56+
accessToken:
57+
title: Access Token
58+
description: The personal access token to access Databricks.
59+
type: string
60+
format: password
61+
x-descriptors:
62+
- urn:camel:group:credentials
63+
query:
64+
title: Query
65+
description: The query to execute against the Databricks Database.
66+
type: string
67+
example: 'INSERT INTO accounts (username,city) VALUES (:#username,:#city)'
68+
extraOptions:
69+
title: Extra Options
70+
description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default).
71+
type: string
72+
default: ''
73+
types:
74+
in:
75+
mediaType: application/json
76+
dependencies:
77+
- "camel:jackson"
78+
- "camel:kamelet"
79+
- "camel:sql"
80+
- "mvn:com.databricks:databricks-jdbc:3.0.7"
81+
- "mvn:org.apache.commons:commons-dbcp2:2.14.0"
82+
template:
83+
beans:
84+
- name: local-sql-databricks-sink
85+
type: "#class:org.apache.camel.component.sql.SqlComponent"
86+
properties:
87+
autowiredEnabled: "false"
88+
- name: dsBean
89+
type: "#class:org.apache.commons.dbcp2.BasicDataSource"
90+
properties:
91+
password: '{{accessToken}}'
92+
url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
93+
driverClassName: 'com.databricks.client.jdbc.Driver'
94+
from:
95+
uri: "kamelet:source"
96+
steps:
97+
- unmarshal:
98+
json:
99+
library: Jackson
100+
- to:
101+
uri: "{{local-sql-databricks-sink}}:{{query}}"
102+
parameters:
103+
dataSource: "#bean:{{dsBean}}"

‎kamelets/databricks-source.kamelet.yaml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# ---------------------------------------------------------------------------
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# ---------------------------------------------------------------------------
17+
apiVersion: camel.apache.org/v1
18+
kind: Kamelet
19+
metadata:
20+
name: databricks-source
21+
annotations:
22+
camel.apache.org/kamelet.support.level: "Preview"
23+
camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
24+
camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMCAyNi45VjMzLjNMMCAyNS45VjE5LjVaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDE5LjVMMjAgMjYuOVYzMy4zTDQwIDI1LjlWMTkuNVoiLz4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMCAyNS45TDIwIDMzLjNWNDBMMCAzMi42VjI1LjlaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDI1LjlMMjAgMzMuM1Y0MEw0MCAzMi42VjI1LjlaIi8+Cjwvc3ZnPgo="
25+
camel.apache.org/provider: "Apache Software Foundation"
26+
camel.apache.org/kamelet.group: "SQL"
27+
camel.apache.org/kamelet.namespace: "Database"
28+
labels:
29+
camel.apache.org/kamelet.type: "source"
30+
spec:
31+
definition:
32+
title: "Databricks Source"
33+
description: Query data from a Databricks Database. For Unity Catalog workspaces, specify catalog and schema parameters.
34+
required:
35+
- serverHostname
36+
- httpPath
37+
- accessToken
38+
- query
39+
type: object
40+
properties:
41+
serverHostname:
42+
title: Server Hostname
43+
description: The Databricks server hostname.
44+
type: string
45+
example: adb-1234567890123456.7.azuredatabricks.net
46+
serverPort:
47+
title: Server Port
48+
description: The server port for the Databricks data source.
49+
type: string
50+
default: 443
51+
httpPath:
52+
title: HTTP Path
53+
description: The HTTP path to the Databricks SQL Warehouse or cluster.
54+
type: string
55+
example: /sql/1.0/warehouses/abc123def456
56+
accessToken:
57+
title: Access Token
58+
description: The personal access token to access Databricks.
59+
type: string
60+
format: password
61+
x-descriptors:
62+
- urn:camel:group:credentials
63+
query:
64+
title: Query
65+
description: The query to execute against the Databricks Database.
66+
type: string
67+
example: 'SELECT * FROM accounts'
68+
extraOptions:
69+
title: Extra Options
70+
description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default).
71+
type: string
72+
default: ''
73+
consumedQuery:
74+
title: Consumed Query
75+
description: A query to run on a tuple consumed.
76+
type: string
77+
example: 'DELETE FROM accounts where user_id = :#user_id'
78+
delay:
79+
title: Delay
80+
description: The number of milliseconds before the next poll from the Databricks database.
81+
type: integer
82+
default: 500
83+
types:
84+
out:
85+
mediaType: application/json
86+
dependencies:
87+
- "camel:jackson"
88+
- "camel:kamelet"
89+
- "camel:sql"
90+
- "mvn:com.databricks:databricks-jdbc:3.0.7"
91+
- "mvn:org.apache.commons:commons-dbcp2:2.14.0"
92+
template:
93+
beans:
94+
- name: local-sql-databricks-source
95+
type: "#class:org.apache.camel.component.sql.SqlComponent"
96+
properties:
97+
autowiredEnabled: "false"
98+
- name: dsBean
99+
type: "#class:org.apache.commons.dbcp2.BasicDataSource"
100+
properties:
101+
password: '{{accessToken}}'
102+
url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
103+
driverClassName: 'com.databricks.client.jdbc.Driver'
104+
from:
105+
uri: "{{local-sql-databricks-source}}:{{query}}"
106+
parameters:
107+
dataSource: "#bean:{{dsBean}}"
108+
onConsume: "{{?consumedQuery}}"
109+
delay: "{{delay}}"
110+
steps:
111+
- marshal:
112+
json:
113+
library: Jackson
114+
- to: "kamelet:sink"

‎library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# ---------------------------------------------------------------------------
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# ---------------------------------------------------------------------------
17+
apiVersion: camel.apache.org/v1
18+
kind: Kamelet
19+
metadata:
20+
name: databricks-sink
21+
annotations:
22+
camel.apache.org/kamelet.support.level: "Preview"
23+
camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
24+
camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMCAyNi45VjMzLjNMMCAyNS45VjE5LjVaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDE5LjVMMjAgMjYuOVYzMy4zTDQwIDI1LjlWMTkuNVoiLz4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMCAyNS45TDIwIDMzLjNWNDBMMCAzMi42VjI1LjlaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTQwIDI1LjlMMjAgMzMuM1Y0MEw0MCAzMi42VjI1LjlaIi8+Cjwvc3ZnPgo="
25+
camel.apache.org/provider: "Apache Software Foundation"
26+
camel.apache.org/kamelet.group: "SQL"
27+
camel.apache.org/kamelet.namespace: "Database"
28+
labels:
29+
camel.apache.org/kamelet.type: "sink"
30+
spec:
31+
definition:
32+
title: "Databricks Sink"
33+
description: Send data to a Databricks Database. This Kamelet expects a JSON-formatted body. Use key:value pairs to map the JSON fields and parameters. For Unity Catalog workspaces, specify catalog and schema parameters.
34+
required:
35+
- serverHostname
36+
- httpPath
37+
- accessToken
38+
- query
39+
type: object
40+
properties:
41+
serverHostname:
42+
title: Server Hostname
43+
description: The Databricks server hostname.
44+
type: string
45+
example: adb-1234567890123456.7.azuredatabricks.net
46+
serverPort:
47+
title: Server Port
48+
description: The server port for the Databricks data source.
49+
type: string
50+
default: 443
51+
httpPath:
52+
title: HTTP Path
53+
description: The HTTP path to the Databricks SQL Warehouse or cluster.
54+
type: string
55+
example: /sql/1.0/warehouses/abc123def456
56+
accessToken:
57+
title: Access Token
58+
description: The personal access token to access Databricks.
59+
type: string
60+
format: password
61+
x-descriptors:
62+
- urn:camel:group:credentials
63+
query:
64+
title: Query
65+
description: The query to execute against the Databricks Database.
66+
type: string
67+
example: 'INSERT INTO accounts (username,city) VALUES (:#username,:#city)'
68+
extraOptions:
69+
title: Extra Options
70+
description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default).
71+
type: string
72+
default: ''
73+
types:
74+
in:
75+
mediaType: application/json
76+
dependencies:
77+
- "camel:jackson"
78+
- "camel:kamelet"
79+
- "camel:sql"
80+
- "mvn:com.databricks:databricks-jdbc:3.0.7"
81+
- "mvn:org.apache.commons:commons-dbcp2:2.14.0"
82+
template:
83+
beans:
84+
- name: local-sql-databricks-sink
85+
type: "#class:org.apache.camel.component.sql.SqlComponent"
86+
properties:
87+
autowiredEnabled: "false"
88+
- name: dsBean
89+
type: "#class:org.apache.commons.dbcp2.BasicDataSource"
90+
properties:
91+
password: '{{accessToken}}'
92+
url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
93+
driverClassName: 'com.databricks.client.jdbc.Driver'
94+
from:
95+
uri: "kamelet:source"
96+
steps:
97+
- unmarshal:
98+
json:
99+
library: Jackson
100+
- to:
101+
uri: "{{local-sql-databricks-sink}}:{{query}}"
102+
parameters:
103+
dataSource: "#bean:{{dsBean}}"

0 commit comments

Comments
 (0)