From 378ce2d4020e2474240f0cd19fe0c857e3e68e3a Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 31 Jul 2020 08:05:21 -0400 Subject: [PATCH 01/21] set a timestamp of when metric was pulled / parsed --- set.go | 1 + 1 file changed, 1 insertion(+) diff --git a/set.go b/set.go index a99216c..e228fe2 100644 --- a/set.go +++ b/set.go @@ -77,6 +77,7 @@ func setValueForResult(r prometheus.Gauge, v interface{}) error { default: return fmt.Errorf("Unhandled type %s", t) } + r.SetToCurrentTime() return nil } From 02b1bb71a8411916086cd656b17879f2ecbf3d2c Mon Sep 17 00:00:00 2001 From: schou Date: Thu, 29 Oct 2020 23:55:15 -0400 Subject: [PATCH 02/21] Allow upper case in labels and add the ability to do handle sql histograms queries --- config.go | 1 - set.go | 58 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/config.go b/config.go index f8f0b1a..24056a3 100644 --- a/config.go +++ b/config.go @@ -193,7 +193,6 @@ func decodeQueries(r io.Reader, config *Config) (QueryList, error) { if q.ValueOnError == "" && config.Defaults.QueryValueOnError != "" { q.ValueOnError = config.Defaults.QueryValueOnError } - q.DataField = strings.ToLower(q.DataField) if err := validateQuery(q); err != nil { return nil, err } diff --git a/set.go b/set.go index e228fe2..19d92a2 100644 --- a/set.go +++ b/set.go @@ -43,7 +43,7 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin resultKey := fmt.Sprintf("%s%s", metricName, string(jsonData)) for k, v := range facets { - labels[k] = strings.ToLower(fmt.Sprintf("%v", v)) + labels[k] = fmt.Sprintf("%v", v) } if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered @@ -107,17 +107,29 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) dataVal interface{} dataFound bool ) + histogram_data := make(map[string]interface{}) + histogram := (datafield[len(datafield)-1:] == "#") for k, v := range row { - if len(row) > 1 && strings.ToLower(k) != datafield { // facet field, add to facets - submetric := false - for _, n := range submetrics { - if strings.ToLower(k) == n { - submetric = true + if len(row) > 1 && k != datafield { + k_str := fmt.Sprintf("%v", k) + if histogram && strings.HasPrefix(k_str, datafield) { + // histogram field, add to histogram_data + histogram_data[k[len(datafield):]] = v + dataFound = true + } else { + // facet field, add to facets + submetric := false + for _, n := range submetrics { + if k == n { + submetric = true + } else if strings.Contains(n, "#") && strings.HasPrefix(k, n) { + submetric = true + } + } + // it is a facet field and not a submetric field + if !submetric { + facet[k_str] = v } - } - // it is a facet field and not a submetric field - if !submetric { - facet[strings.ToLower(fmt.Sprintf("%v", k))] = v } } else { // this is the actual gauge data if dataFound { @@ -132,12 +144,28 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) return nil, errors.New("Data field not found in result set") } - key, status := r.registerMetric(facet, suffix) - err := setValueForResult(r.Result[key], dataVal) - if err != nil { - return nil, err + if histogram { + histogram_field := datafield[0 : len(datafield)-1] + for k, v := range histogram_data { + // loop over histogram data registering bins + dataVal = v + facet[histogram_field] = k + + key, status := r.registerMetric(facet, suffix) + err := setValueForResult(r.Result[key], dataVal) + if err != nil { + return nil, err + } + facetsWithResult[key] = status + } + } else { + key, status := r.registerMetric(facet, suffix) + err := setValueForResult(r.Result[key], dataVal) + if err != nil { + return nil, err + } + facetsWithResult[key] = status } - facetsWithResult[key] = status } } From d7a13c6074c1d427869d2907427cd82c5554f791 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 08:42:31 -0400 Subject: [PATCH 03/21] set the labels to have lower first letter --- set.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/set.go b/set.go index 19d92a2..4327363 100644 --- a/set.go +++ b/set.go @@ -43,7 +43,7 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin resultKey := fmt.Sprintf("%s%s", metricName, string(jsonData)) for k, v := range facets { - labels[k] = fmt.Sprintf("%v", v) + labels[k] = labelCaseChange(fmt.Sprintf("%v", v)) } if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered @@ -107,12 +107,13 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) dataVal interface{} dataFound bool ) + datafield = labelCaseChange(datafield) histogram_data := make(map[string]interface{}) histogram := (datafield[len(datafield)-1:] == "#") for k, v := range row { if len(row) > 1 && k != datafield { - k_str := fmt.Sprintf("%v", k) - if histogram && strings.HasPrefix(k_str, datafield) { + k := labelCaseChange(fmt.Sprintf("%v", k)) + if histogram && strings.HasPrefix(k, datafield) { // histogram field, add to histogram_data histogram_data[k[len(datafield):]] = v dataFound = true @@ -120,20 +121,20 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // facet field, add to facets submetric := false for _, n := range submetrics { - if k == n { + if k == labelCaseChange(n) { submetric = true - } else if strings.Contains(n, "#") && strings.HasPrefix(k, n) { + } else if strings.Contains(n, "#") && strings.HasPrefix(k, labelCaseChange(n)) { submetric = true } } // it is a facet field and not a submetric field if !submetric { - facet[k_str] = v + facet[k] = v } } } else { // this is the actual gauge data if dataFound { - return nil, errors.New("Data field not specified for multi-column query") + return nil, errors.New(fmt.Sprintf("Data field '%v' not specified for multi-column query", datafield)) } dataVal = v dataFound = true @@ -141,7 +142,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) } if !dataFound { - return nil, errors.New("Data field not found in result set") + return nil, errors.New(fmt.Sprintf("Data field '%v' not found in result set", datafield)) } if histogram { @@ -189,3 +190,6 @@ func (r *QueryResult) RegisterMetrics(facetsWithResult map[string]metricStatus) } } } +func labelCaseChange(str string) string { + return string(strings.ToLower(str[0:1])) + str[1:] +} From 30a86410a87c920a960b08f2a0a25521507a3d06 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 09:09:13 -0400 Subject: [PATCH 04/21] undo case change on values --- set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/set.go b/set.go index 4327363..d96114b 100644 --- a/set.go +++ b/set.go @@ -43,7 +43,7 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin resultKey := fmt.Sprintf("%s%s", metricName, string(jsonData)) for k, v := range facets { - labels[k] = labelCaseChange(fmt.Sprintf("%v", v)) + labels[k] = fmt.Sprintf("%v", v) } if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered From a393b9ce2b3230f548050e2f0bc3c0290ffe9b42 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 09:19:01 -0400 Subject: [PATCH 05/21] need to keep type as interface --- set.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/set.go b/set.go index d96114b..6907595 100644 --- a/set.go +++ b/set.go @@ -147,9 +147,8 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) if histogram { histogram_field := datafield[0 : len(datafield)-1] - for k, v := range histogram_data { + for k, dataVal := range histogram_data { // loop over histogram data registering bins - dataVal = v facet[histogram_field] = k key, status := r.registerMetric(facet, suffix) From 3e5b411ca52410de4517dcbb0cab824f02e00b3c Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 10:33:07 -0400 Subject: [PATCH 06/21] remove a debug test value --- Makefile | 2 +- examples/example-config-queries.yml | 27 +++++++++++++++++++++++++++ set.go | 1 - 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bb134a5..762c9a9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ GIT_BRANCH := $(shell git symbolic-ref -q --short HEAD) GIT_VERSION := $(shell git log -1 --pretty=format:"%h (%ci)" .) build: - go build -ldflags "-X \"main.buildVersion=$(GIT_VERSION)\"" \ + CGO_ENABLED=0 go build -ldflags "-X \"main.buildVersion=$(GIT_VERSION)\"" \ -o $(GOPATH)/bin/$(PROG_NAME) $(CMD_PATH) dist-build: diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index a4fc048..35b5d08 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -19,6 +19,7 @@ # insert into Companies (name, country) values ('Company1', 'IRL'); # insert into Companies (name, country) values ('Company2', 'IRL'); # +# # select * from Companies; # +----------+---------+ # | name | country | @@ -30,6 +31,19 @@ # +----------+---------+ # 4 rows in set (0.00 sec) # +# -- Should you want to change the column name or upper case / lower case a column use 'as' +# -- note: the first character must be a lowercase character for grafana +# select name as companyName from Companies; +# +-------------+ +# | companyName | +# +-------------+ +# | Company1 | +# | Company1 | +# | Company1 | +# | Company2 | +# +-------------+ +# 4 rows in set (0.00 sec) +# # quit # # docker run -d -p 8080:8080 -v ${PWD}/example-config-queries.yml:/queries.yml -v ${PWD}/example-config.yml:/prometheus-sql.yml --link sqlagent:sqlagent --name prometheus-sql dbhi/prometheus-sql -service http://sqlagent:5000 -config prometheus-sql.yml @@ -64,3 +78,16 @@ count: cnt sum: rt interval: 30s + +# Histogram queries +# This will register an array of metrics intended for a histogram: +# - responseHistogram with label values of 0,1,2, and 3 representing the bin for seconds taken +- response_time_histogram: + sql: > + select count(CASE WHEN response_time >= 0 AND response_time < 1 THEN 1 END) as 'responseHistogram#0', + select count(CASE WHEN response_time >= 1 AND response_time < 2 THEN 1 END) as 'responseHistogram#1', + select count(CASE WHEN response_time >= 2 AND response_time < 3 THEN 1 END) as 'responseHistogram#2', + select count(CASE WHEN response_time >= 3 THEN 1 END) as 'responseHistogram#3' + from stats + data-field: 'responseHistogram#' + interval: 30s diff --git a/set.go b/set.go index 6907595..95b103c 100644 --- a/set.go +++ b/set.go @@ -77,7 +77,6 @@ func setValueForResult(r prometheus.Gauge, v interface{}) error { default: return fmt.Errorf("Unhandled type %s", t) } - r.SetToCurrentTime() return nil } From 06ebaa0340854f9f098ef019d80ad4e2b2aa7388 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 10:34:12 -0400 Subject: [PATCH 07/21] add comments on how to use changes, updated dockerfile to complie a binary that fits well in a container --- Dockerfile | 2 +- examples/example-config-queries.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a2f0e18..0b33efd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,4 @@ COPY --from=builder /go/bin/app /usr/local/bin/prometheus-sql EXPOSE 8080 ENTRYPOINT ["/usr/local/bin/prometheus-sql", "-host", "0.0.0.0"] # Default command assumes the SQL agent is linked. -CMD ["-service", "http://sqlagent:5000"] \ No newline at end of file +CMD ["-service", "http://sqlagent:5000"] diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 35b5d08..fd761b7 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -91,3 +91,17 @@ from stats data-field: 'responseHistogram#' interval: 30s + +# Hard setting a field for easy labeling +# This will register an extra field intended to be used for labeling such as the case when two or +# more databases have similar records and you with to set a custom field for identifying the database: +# - last_login_time is generated with overlapping metric label 'systemName' +- last_login_time: + sql: > + select *, 'user_portal_database' as 'systemName' from logins + interval: 30s +- last_login_time: + sql: > + select *, 'user_registration_database' as 'systemName' from logins + interval: 30s + From 982175a7eafefccb8f0d99b78e3537de2d78b957 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 11:46:50 -0400 Subject: [PATCH 08/21] add the option to preserve case or leave lower --- config.go | 1 + set.go | 29 +++++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/config.go b/config.go index 24056a3..b04d1ec 100644 --- a/config.go +++ b/config.go @@ -56,6 +56,7 @@ type Query struct { Params map[string]interface{} Interval time.Duration Timeout time.Duration + PreserveCase bool `yaml:"keep-case"` DataField string `yaml:"data-field"` SubMetrics map[string]string `yaml:"sub-metrics"` ValueOnError string `yaml:"value-on-error"` diff --git a/set.go b/set.go index 95b103c..e9c930d 100644 --- a/set.go +++ b/set.go @@ -32,7 +32,7 @@ func NewQueryResult(q *Query) *QueryResult { return r } -func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string) (string, metricStatus) { +func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string, keepCase bool) (string, metricStatus) { labels := prometheus.Labels{} metricName := r.Query.Name if suffix != "" { @@ -43,7 +43,11 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin resultKey := fmt.Sprintf("%s%s", metricName, string(jsonData)) for k, v := range facets { - labels[k] = fmt.Sprintf("%v", v) + if keepCase { + labels[k] = fmt.Sprintf("%v", v) + } else { + labels[k] = strings.ToLower(fmt.Sprintf("%v", v)) + } } if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered @@ -91,6 +95,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) } submetrics := map[string]string{} + keepCase := (r.Query.PreserveCase == true) if len(r.Query.SubMetrics) > 0 { submetrics = r.Query.SubMetrics @@ -106,12 +111,12 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) dataVal interface{} dataFound bool ) - datafield = labelCaseChange(datafield) + datafield = labelCaseChange(datafield, keepCase) histogram_data := make(map[string]interface{}) histogram := (datafield[len(datafield)-1:] == "#") for k, v := range row { if len(row) > 1 && k != datafield { - k := labelCaseChange(fmt.Sprintf("%v", k)) + k := labelCaseChange(fmt.Sprintf("%v", k), keepCase) if histogram && strings.HasPrefix(k, datafield) { // histogram field, add to histogram_data histogram_data[k[len(datafield):]] = v @@ -120,9 +125,9 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // facet field, add to facets submetric := false for _, n := range submetrics { - if k == labelCaseChange(n) { + if k == labelCaseChange(n, keepCase) { submetric = true - } else if strings.Contains(n, "#") && strings.HasPrefix(k, labelCaseChange(n)) { + } else if strings.Contains(n, "#") && strings.HasPrefix(k, labelCaseChange(n, keepCase)) { submetric = true } } @@ -150,7 +155,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // loop over histogram data registering bins facet[histogram_field] = k - key, status := r.registerMetric(facet, suffix) + key, status := r.registerMetric(facet, suffix, keepCase) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err @@ -158,7 +163,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) facetsWithResult[key] = status } } else { - key, status := r.registerMetric(facet, suffix) + key, status := r.registerMetric(facet, suffix, keepCase) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err @@ -188,6 +193,10 @@ func (r *QueryResult) RegisterMetrics(facetsWithResult map[string]metricStatus) } } } -func labelCaseChange(str string) string { - return string(strings.ToLower(str[0:1])) + str[1:] +func labelCaseChange(str string, keepCase bool) string { + if keepCase { + return string(strings.ToLower(str[0:1])) + str[1:] + } else { + return strings.ToLower(str) + } } From 5d4a93304781ff940a6fbf52ecb919edb6aad7c2 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 11:56:51 -0400 Subject: [PATCH 09/21] updating the example for histogram --- examples/example-config-queries.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index fd761b7..75ed3ea 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -82,14 +82,18 @@ # Histogram queries # This will register an array of metrics intended for a histogram: # - responseHistogram with label values of 0,1,2, and 3 representing the bin for seconds taken -- response_time_histogram: +- response_times: sql: > - select count(CASE WHEN response_time >= 0 AND response_time < 1 THEN 1 END) as 'responseHistogram#0', - select count(CASE WHEN response_time >= 1 AND response_time < 2 THEN 1 END) as 'responseHistogram#1', - select count(CASE WHEN response_time >= 2 AND response_time < 3 THEN 1 END) as 'responseHistogram#2', - select count(CASE WHEN response_time >= 3 THEN 1 END) as 'responseHistogram#3' + select count(CASE WHEN response_time >= 0 AND response_time <= 1 THEN 1 END) as 'le#1', + count(CASE WHEN response_time > 1 AND response_time <= 2 THEN 1 END) as 'le#2', + count(CASE WHEN response_time > 2 AND response_time <= 3 THEN 1 END) as 'le#3', + count(CASE WHEN response_time > 3 THEN 1 END) as 'le#+Inf', + sum(response_time) s, count(response_time) c from stats - data-field: 'responseHistogram#' + sub-metrics: + bucket: 'responseHistogram#' + total: 't' + count: 'c' interval: 30s # Hard setting a field for easy labeling From 79ef49e6e7681d622a2dd0ce6ab65d7873e11863 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 30 Oct 2020 15:13:57 -0400 Subject: [PATCH 10/21] add example for config to keep case --- examples/example-config-queries.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 75ed3ea..86af6f2 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -88,11 +88,11 @@ count(CASE WHEN response_time > 1 AND response_time <= 2 THEN 1 END) as 'le#2', count(CASE WHEN response_time > 2 AND response_time <= 3 THEN 1 END) as 'le#3', count(CASE WHEN response_time > 3 THEN 1 END) as 'le#+Inf', - sum(response_time) s, count(response_time) c + sum(response_time) as s, count(response_time) as c from stats sub-metrics: - bucket: 'responseHistogram#' - total: 't' + bucket: 'le#' + total: 's' count: 'c' interval: 30s @@ -100,11 +100,14 @@ # This will register an extra field intended to be used for labeling such as the case when two or # more databases have similar records and you with to set a custom field for identifying the database: # - last_login_time is generated with overlapping metric label 'systemName' +# - keep-case will allow uppercase keys and values to be passed to prometheus - last_login_time: + keep-case: true sql: > select *, 'user_portal_database' as 'systemName' from logins interval: 30s - last_login_time: + keep-case: true sql: > select *, 'user_registration_database' as 'systemName' from logins interval: 30s From 3b46bd40eb56294a295254c866c6f1258a397f81 Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 6 Nov 2020 13:21:39 -0500 Subject: [PATCH 11/21] adding label: field to config --- config.go | 1 + examples/example-config-queries.yml | 4 +++- set.go | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index b04d1ec..bcc5e82 100644 --- a/config.go +++ b/config.go @@ -59,6 +59,7 @@ type Query struct { PreserveCase bool `yaml:"keep-case"` DataField string `yaml:"data-field"` SubMetrics map[string]string `yaml:"sub-metrics"` + ExtraLabels map[string]string `yaml:"labels"` ValueOnError string `yaml:"value-on-error"` } diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 86af6f2..84ba4b3 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -108,7 +108,9 @@ interval: 30s - last_login_time: keep-case: true + labels: + systemName: user_registration_database sql: > - select *, 'user_registration_database' as 'systemName' from logins + select * from logins interval: 30s diff --git a/set.go b/set.go index e9c930d..85408a5 100644 --- a/set.go +++ b/set.go @@ -95,6 +95,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) } submetrics := map[string]string{} + extralabels := map[string]string{} keepCase := (r.Query.PreserveCase == true) if len(r.Query.SubMetrics) > 0 { @@ -103,10 +104,17 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) submetrics = map[string]string{"": r.Query.DataField} } + if len(r.Query.ExtraLabels) > 0 { + extralabels = r.Query.ExtraLabels + } + facetsWithResult := make(map[string]metricStatus, 0) for _, row := range recs { for suffix, datafield := range submetrics { facet := make(map[string]interface{}) + for k, v := range extralabels { + facet[k] = v + } var ( dataVal interface{} dataFound bool From 919ea2e06a9f16c6aee93031aec63fad4102aab5 Mon Sep 17 00:00:00 2001 From: schou Date: Sat, 7 Nov 2020 20:28:03 -0500 Subject: [PATCH 12/21] Add the ability to set the help-text in the yaml --- config.go | 1 + examples/example-queries.yml | 3 +++ set.go | 12 ++++++++---- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index bcc5e82..484f086 100644 --- a/config.go +++ b/config.go @@ -58,6 +58,7 @@ type Query struct { Timeout time.Duration PreserveCase bool `yaml:"keep-case"` DataField string `yaml:"data-field"` + HelpText string `yaml:"help-text"` SubMetrics map[string]string `yaml:"sub-metrics"` ExtraLabels map[string]string `yaml:"labels"` ValueOnError string `yaml:"value-on-error"` diff --git a/examples/example-queries.yml b/examples/example-queries.yml index 6a3286d..b0ad065 100644 --- a/examples/example-queries.yml +++ b/examples/example-queries.yml @@ -48,6 +48,9 @@ # Name of the driver to use. driver: postgresql + # Help text about query + help-text: "Result of an SQL query on example.org" + # Connection information. connection: host: example.org diff --git a/set.go b/set.go index 85408a5..d04acb5 100644 --- a/set.go +++ b/set.go @@ -32,7 +32,7 @@ func NewQueryResult(q *Query) *QueryResult { return r } -func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string, keepCase bool) (string, metricStatus) { +func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string, keepCase bool, helpText string) (string, metricStatus) { labels := prometheus.Labels{} metricName := r.Query.Name if suffix != "" { @@ -54,10 +54,14 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin return resultKey, registered } + if len(helpText) == 0 { + helpText = "Result of an SQL query" + } + fmt.Println("Creating", resultKey) r.Result[resultKey] = prometheus.NewGauge(prometheus.GaugeOpts{ Name: fmt.Sprintf("query_result_%s", metricName), - Help: "Result of an SQL query", + Help: helpText, ConstLabels: labels, }) return resultKey, unregistered @@ -163,7 +167,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // loop over histogram data registering bins facet[histogram_field] = k - key, status := r.registerMetric(facet, suffix, keepCase) + key, status := r.registerMetric(facet, suffix, keepCase, r.Query.HelpText) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err @@ -171,7 +175,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) facetsWithResult[key] = status } } else { - key, status := r.registerMetric(facet, suffix, keepCase) + key, status := r.registerMetric(facet, suffix, keepCase, r.Query.HelpText) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err From 8959c314ba870854b6b7ddd43afcd9e4460cd3cc Mon Sep 17 00:00:00 2001 From: schou Date: Sun, 8 Nov 2020 19:46:14 -0500 Subject: [PATCH 13/21] add options for case, lower, upper, title, first, and keep --- config.go | 3 ++- examples/example-config-queries.yml | 14 ++++++++--- set.go | 39 ++++++++++++++++------------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/config.go b/config.go index 484f086..5ce2112 100644 --- a/config.go +++ b/config.go @@ -56,7 +56,8 @@ type Query struct { Params map[string]interface{} Interval time.Duration Timeout time.Duration - PreserveCase bool `yaml:"keep-case"` + ValueCase string `yaml:"value-case"` + LabelCase string `yaml:"label-case"` DataField string `yaml:"data-field"` HelpText string `yaml:"help-text"` SubMetrics map[string]string `yaml:"sub-metrics"` diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 84ba4b3..3c5f294 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -100,14 +100,22 @@ # This will register an extra field intended to be used for labeling such as the case when two or # more databases have similar records and you with to set a custom field for identifying the database: # - last_login_time is generated with overlapping metric label 'systemName' -# - keep-case will allow uppercase keys and values to be passed to prometheus +# - label-case and value-case will alter the case of keys and values before being passed to prometheus +# possible values for label-case and value-case are: +# - lower - lower case of the string ('MyVal' -> 'myval') -- default -- +# - upper - upper case the string ('MyVal' -> 'MYVAL') +# - first - lower case the initial character ('MyVal' -> 'myVal') +# - title - upper case the initial character ('myval' -> 'Myval') +# - keep - leave the case alone and pass as untouched ('MyVal' -> 'MyVal') - last_login_time: - keep-case: true + label-case: lower + value-case: lower sql: > select *, 'user_portal_database' as 'systemName' from logins interval: 30s - last_login_time: - keep-case: true + label-case: first + value-case: keep labels: systemName: user_registration_database sql: > diff --git a/set.go b/set.go index d04acb5..fb33b3d 100644 --- a/set.go +++ b/set.go @@ -32,7 +32,7 @@ func NewQueryResult(q *Query) *QueryResult { return r } -func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string, keepCase bool, helpText string) (string, metricStatus) { +func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix string, valueCase string, helpText string) (string, metricStatus) { labels := prometheus.Labels{} metricName := r.Query.Name if suffix != "" { @@ -43,11 +43,7 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin resultKey := fmt.Sprintf("%s%s", metricName, string(jsonData)) for k, v := range facets { - if keepCase { - labels[k] = fmt.Sprintf("%v", v) - } else { - labels[k] = strings.ToLower(fmt.Sprintf("%v", v)) - } + labels[k] = CaseChange(fmt.Sprintf("%v", v), valueCase) } if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered @@ -100,7 +96,8 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) submetrics := map[string]string{} extralabels := map[string]string{} - keepCase := (r.Query.PreserveCase == true) + labelCase := r.Query.LabelCase + valueCase := r.Query.ValueCase if len(r.Query.SubMetrics) > 0 { submetrics = r.Query.SubMetrics @@ -123,12 +120,12 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) dataVal interface{} dataFound bool ) - datafield = labelCaseChange(datafield, keepCase) + datafield = CaseChange(datafield, labelCase) histogram_data := make(map[string]interface{}) histogram := (datafield[len(datafield)-1:] == "#") for k, v := range row { if len(row) > 1 && k != datafield { - k := labelCaseChange(fmt.Sprintf("%v", k), keepCase) + k := CaseChange(fmt.Sprintf("%v", k), labelCase) if histogram && strings.HasPrefix(k, datafield) { // histogram field, add to histogram_data histogram_data[k[len(datafield):]] = v @@ -137,9 +134,9 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // facet field, add to facets submetric := false for _, n := range submetrics { - if k == labelCaseChange(n, keepCase) { + if k == CaseChange(n, labelCase) { submetric = true - } else if strings.Contains(n, "#") && strings.HasPrefix(k, labelCaseChange(n, keepCase)) { + } else if strings.Contains(n, "#") && strings.HasPrefix(k, CaseChange(n, labelCase)) { submetric = true } } @@ -167,7 +164,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) // loop over histogram data registering bins facet[histogram_field] = k - key, status := r.registerMetric(facet, suffix, keepCase, r.Query.HelpText) + key, status := r.registerMetric(facet, suffix, valueCase, r.Query.HelpText) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err @@ -175,7 +172,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) facetsWithResult[key] = status } } else { - key, status := r.registerMetric(facet, suffix, keepCase, r.Query.HelpText) + key, status := r.registerMetric(facet, suffix, valueCase, r.Query.HelpText) err := setValueForResult(r.Result[key], dataVal) if err != nil { return nil, err @@ -205,10 +202,18 @@ func (r *QueryResult) RegisterMetrics(facetsWithResult map[string]metricStatus) } } } -func labelCaseChange(str string, keepCase bool) string { - if keepCase { - return string(strings.ToLower(str[0:1])) + str[1:] - } else { +func CaseChange(str string, newCase string) string { + switch newCase { + case "lower": return strings.ToLower(str) + case "upper": + return strings.ToUpper(str) + case "first": + return string(strings.ToLower(str[0:1])) + str[1:] + case "title": + return string(strings.ToUpper(str[0:1])) + str[1:] + case "keep": + return str } + return strings.ToLower(str) } From 54956da1cd6fc2af384b90790667d8ba20220719 Mon Sep 17 00:00:00 2001 From: schou Date: Sun, 8 Nov 2020 19:54:33 -0500 Subject: [PATCH 14/21] grammer, removing of --- examples/example-config-queries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 3c5f294..9268e0e 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -102,7 +102,7 @@ # - last_login_time is generated with overlapping metric label 'systemName' # - label-case and value-case will alter the case of keys and values before being passed to prometheus # possible values for label-case and value-case are: -# - lower - lower case of the string ('MyVal' -> 'myval') -- default -- +# - lower - lower case the string ('MyVal' -> 'myval') -- default -- # - upper - upper case the string ('MyVal' -> 'MYVAL') # - first - lower case the initial character ('MyVal' -> 'myVal') # - title - upper case the initial character ('myval' -> 'Myval') From 352695e53c281901513f0d0242ba30739319cd5e Mon Sep 17 00:00:00 2001 From: schou Date: Sun, 8 Nov 2020 19:54:33 -0500 Subject: [PATCH 15/21] grammar, removing of --- examples/example-config-queries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 3c5f294..9268e0e 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -102,7 +102,7 @@ # - last_login_time is generated with overlapping metric label 'systemName' # - label-case and value-case will alter the case of keys and values before being passed to prometheus # possible values for label-case and value-case are: -# - lower - lower case of the string ('MyVal' -> 'myval') -- default -- +# - lower - lower case the string ('MyVal' -> 'myval') -- default -- # - upper - upper case the string ('MyVal' -> 'MYVAL') # - first - lower case the initial character ('MyVal' -> 'myVal') # - title - upper case the initial character ('myval' -> 'Myval') From a60cf269cb51715a352afe2aaf2b444ce54b8ae4 Mon Sep 17 00:00:00 2001 From: schou Date: Sun, 8 Nov 2020 20:39:59 -0500 Subject: [PATCH 16/21] removed reference to grafana --- examples/example-config-queries.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/example-config-queries.yml b/examples/example-config-queries.yml index 9268e0e..2b5a443 100644 --- a/examples/example-config-queries.yml +++ b/examples/example-config-queries.yml @@ -32,7 +32,6 @@ # 4 rows in set (0.00 sec) # # -- Should you want to change the column name or upper case / lower case a column use 'as' -# -- note: the first character must be a lowercase character for grafana # select name as companyName from Companies; # +-------------+ # | companyName | From 42e5e6e596bc0a7a44ab5443468725e6d5d9377a Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 20 Nov 2020 22:04:01 -0500 Subject: [PATCH 17/21] update the set to handle nil values and set them as nan --- set.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/set.go b/set.go index fb33b3d..913e946 100644 --- a/set.go +++ b/set.go @@ -4,10 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "github.com/prometheus/client_golang/prometheus" + "math" "strconv" "strings" - - "github.com/prometheus/client_golang/prometheus" ) type metricStatus int @@ -68,6 +68,8 @@ type records []record func setValueForResult(r prometheus.Gauge, v interface{}) error { switch t := v.(type) { + case nil: + r.Set(math.NaN()) case string: f, err := strconv.ParseFloat(t, 64) if err != nil { From 3978dea4ef630a6cb808912346b2da2cdcdb08cb Mon Sep 17 00:00:00 2001 From: schou Date: Sat, 21 Nov 2020 15:50:24 -0500 Subject: [PATCH 18/21] revert Makefile and Dockerfile to the original --- Dockerfile | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0b33efd..76de303 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,3 +14,4 @@ EXPOSE 8080 ENTRYPOINT ["/usr/local/bin/prometheus-sql", "-host", "0.0.0.0"] # Default command assumes the SQL agent is linked. CMD ["-service", "http://sqlagent:5000"] + diff --git a/Makefile b/Makefile index 762c9a9..bb134a5 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ GIT_BRANCH := $(shell git symbolic-ref -q --short HEAD) GIT_VERSION := $(shell git log -1 --pretty=format:"%h (%ci)" .) build: - CGO_ENABLED=0 go build -ldflags "-X \"main.buildVersion=$(GIT_VERSION)\"" \ + go build -ldflags "-X \"main.buildVersion=$(GIT_VERSION)\"" \ -o $(GOPATH)/bin/$(PROG_NAME) $(CMD_PATH) dist-build: From 9d381def39de89f91d4eb4770db82ee84d0db9b8 Mon Sep 17 00:00:00 2001 From: schou Date: Sat, 21 Nov 2020 15:52:17 -0500 Subject: [PATCH 19/21] revert Docker file --- Dockerfile | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76de303..34e070a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,22 @@ # Requires Docker v17.06 or later -FROM golang:1.9 as builder +FROM golang:1.15.5 as builder RUN mkdir -p /go/src/app WORKDIR /go/src/app COPY . /go/src/app -RUN go-wrapper download -u github.com/golang/dep/cmd/dep -RUN go-wrapper install github.com/golang/dep/cmd/dep -RUN dep ensure -RUN go-wrapper install +RUN go build -v . + +FROM frolvlad/alpine-glibc:alpine-3.12_glibc-2.32 + +ENV PROMSQL_BIND_ADDRESS="0.0.0.0" +ENV PROMSQL_PORT="8080" + +COPY --from=builder /go/src/app/prometheus-sql /usr/local/bin/prometheus-sql +COPY docker-entrypoint.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/* -FROM frolvlad/alpine-glibc:alpine-3.6 -COPY --from=builder /go/bin/app /usr/local/bin/prometheus-sql EXPOSE 8080 -ENTRYPOINT ["/usr/local/bin/prometheus-sql", "-host", "0.0.0.0"] -# Default command assumes the SQL agent is linked. -CMD ["-service", "http://sqlagent:5000"] +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +# Default command assumes the SQL agent is linked. +CMD ["-service", "http://sqlagent:5000"] \ No newline at end of file From 1da0795ab743eb89221a68db51206c8f9e3c5fd8 Mon Sep 17 00:00:00 2001 From: schou Date: Sat, 21 Nov 2020 16:17:51 -0500 Subject: [PATCH 20/21] first pass to merge set.go commits --- set.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/set.go b/set.go index 913e946..d70fa76 100644 --- a/set.go +++ b/set.go @@ -10,6 +10,9 @@ import ( "strings" ) +type record map[string]interface{} +type records []record + type metricStatus int const ( @@ -17,12 +20,13 @@ const ( unregistered ) +// QueryResult contains query results type QueryResult struct { Query *Query Result map[string]prometheus.Gauge // Internally we represent each facet with a JSON-encoded string for simplicity } -// NewSetMetrics initializes a new metrics collector. +// NewQueryMetrics initializes a new metrics collector. func NewQueryResult(q *Query) *QueryResult { r := &QueryResult{ Query: q, @@ -46,7 +50,8 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin labels[k] = CaseChange(fmt.Sprintf("%v", v), valueCase) } - if _, ok := r.Result[resultKey]; ok { // A metric with this name is already registered + if _, ok := r.Result[resultKey]; ok { + // A metric with this key is already created and assumed to be registered return resultKey, registered } @@ -63,9 +68,6 @@ func (r *QueryResult) registerMetric(facets map[string]interface{}, suffix strin return resultKey, unregistered } -type record map[string]interface{} -type records []record - func setValueForResult(r prometheus.Gauge, v interface{}) error { switch t := v.(type) { case nil: @@ -187,6 +189,7 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) return facetsWithResult, nil } +// RegisterMetrics registers and unregister gauges func (r *QueryResult) RegisterMetrics(facetsWithResult map[string]metricStatus) { for key, m := range r.Result { status, ok := facetsWithResult[key] From bfae86a89ee4d02017c3c9a70de74599c7e9e074 Mon Sep 17 00:00:00 2001 From: schou Date: Wed, 2 Dec 2020 12:02:47 -0500 Subject: [PATCH 21/21] change k early and once --- set.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/set.go b/set.go index d70fa76..de4cbc3 100644 --- a/set.go +++ b/set.go @@ -88,6 +88,7 @@ func setValueForResult(r prometheus.Gauge, v interface{}) error { return nil } +// SetMetrics set and register metrics func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) { // Queries that return only one record should only have one column if len(recs) > 1 && len(recs[0]) == 1 { @@ -128,8 +129,8 @@ func (r *QueryResult) SetMetrics(recs records) (map[string]metricStatus, error) histogram_data := make(map[string]interface{}) histogram := (datafield[len(datafield)-1:] == "#") for k, v := range row { + k := CaseChange(fmt.Sprintf("%v", k), labelCase) if len(row) > 1 && k != datafield { - k := CaseChange(fmt.Sprintf("%v", k), labelCase) if histogram && strings.HasPrefix(k, datafield) { // histogram field, add to histogram_data histogram_data[k[len(datafield):]] = v