fix problem with getting URL data (fix #162)

Author: dimus

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+## [v1.1.11] - 2026-05-19 Tue
+
+- Fix [#162]: Fix URL ingestion. It stopped working because sites like
+  Wikipedia became less permissive to empty user agents and requests that
+  do not specify text/http in the header.
+- Fix: prevent crashes caused by "perfect" odds edge case.
+
 ## [v1.1.10] - 2025-04-17 Thu
 
 - Add: slog as as a logging engine.

CITATION.cff

@@ -1,12 +1,12 @@
 cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 title: "GNfinder -- a finder of scientific names in a variety of media."
-version: v1.1.5
+version: v1.1.11
 authors:
   - family-names: "Mozzherin"
     given-names: "Dmitry"
     orcid: "https://orcid.org/0000-0003-1593-1417"
 repository-code: "https://github.com/gnames/gnfinder"
-date-released: 2024-05-24
+date-released: 2026-05-19
 doi: 10.5281/zenodo.10070488
 license: MIT

cmd/root.go

@@ -160,7 +160,8 @@ verification results.
 			input = "STDIN"
 		case 1:
 			input = args[0]
-			d := gndoc.New(cfg.TikaURL)
+			ua := "gnfinder/" + gnfinder.Version + " (https://github.com/gnames/gnfinder)"
+			d := gndoc.New(cfg.TikaURL, gndoc.OptUserAgent(ua))
 			if strings.HasPrefix(input, "http") {
 				data, convDur, err = d.TextFromURL(input)
 			} else {

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/abadojack/whatlanggo v1.0.1
 	github.com/aclements/perflock v0.0.0-20240209181333-b67f3f23152f
 	github.com/gnames/bayes v0.5.3
-	github.com/gnames/gndoc v0.3.2
+	github.com/gnames/gndoc v0.3.3
 	github.com/gnames/gner v0.1.6
 	github.com/gnames/gnfmt v0.6.0
 	github.com/gnames/gnlib v0.47.0

go.sum

@@ -129,8 +129,8 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gnames/bayes v0.5.3 h1:CFr1tYqmvo+VNFKLXQ/Dl58KFFpcnJbFK5csY+w1nI0=
 github.com/gnames/bayes v0.5.3/go.mod h1:tiWMnXQ30NZAyxfwysZsnFPTTehpEwiL3cQThuYF8sc=
-github.com/gnames/gndoc v0.3.2 h1:A7GpdFh6fvjpdisDGwk85p1ihDkpZ59WtPDUfnmT7eU=
-github.com/gnames/gndoc v0.3.2/go.mod h1:SWHl1vRaVlc0E1oftUseKVq6S1GjY8jYYv5tgUBpdy8=
+github.com/gnames/gndoc v0.3.3 h1:TNmHelaWD5RN9Ryra4BMO55/R+ZTmv9O1Bmo1vcQJzg=
+github.com/gnames/gndoc v0.3.3/go.mod h1:SWHl1vRaVlc0E1oftUseKVq6S1GjY8jYYv5tgUBpdy8=
 github.com/gnames/gner v0.1.6 h1:KzAAw2Lxr5GLoh5/ouMEPIuqFzt3TGVgJ0/ny0YgohE=
 github.com/gnames/gner v0.1.6/go.mod h1:GXQJ07xu31O9ufuUBhVE1JKrhc0YUBneNm3CBr82koc=
 github.com/gnames/gnfmt v0.6.0 h1:u+OjRyD8cVywB00IGKf24xTovz9y807E1fAa22+HInc=

pkg/io/web/api.go

@@ -228,8 +228,9 @@ func getText(
 		return params.Text, filename, dur, err
 	}
 
-	d := gndoc.New(tikaURL)
 	if params.URL != "" {
+		ua := "gnfinder/" + gnfinder.Version + " (https://github.com/gnames/gnfinder)"
+		d := gndoc.New(tikaURL, gndoc.OptUserAgent(ua))
 		txt, dur, err = d.TextFromURL(params.URL)
 		return txt, filename, dur, err
 	}

pkg/io/web/web.go

@@ -70,7 +70,8 @@ func find(gnf gnfinder.GNfinder) func(echo.Context) error {
 		if t := params.Text; t != "" {
 			txt = t
 		} else if url := params.URL; url != "" {
-			doc := gndoc.New(tikaURL)
+			ua := "gnfinder/" + gnfinder.Version + " (https://github.com/gnames/gnfinder)"
+			doc := gndoc.New(tikaURL, gndoc.OptUserAgent(ua))
 			txt, dur.TextExtraction, err = doc.TextFromURL(url)
 		} else {
 			txt, filename, dur.TextExtraction, err = textFromFile(c, tikaURL)

pkg/version.go

@@ -1,6 +1,6 @@
 package gnfinder
 
 var (
-	Version = "v1.1.10"
+	Version = "v1.1.11"
 	Build   string
 )