Merge pull request #16 from massix/feat/skip-jpeg-images

Automatically skip JPEG images in CLI and Serve modes

Author: massix

README.md

@@ -55,7 +55,7 @@ Gleeter understands the following commands:
 *   `help`: Prints help information, this is also the default behavior if no arguments are provided.
 *   `version`: Prints the application version.
 *   `latest`: Fetches the latest comic from XKCD and displays it in the terminal.
-*   `random`: Fetches a random comic from XKCD. **Warning**: Some very old comics (IDs ranging from 1 to 120 and possibly more) might not display correctly due to unsupported PNG compression.
+*   `random`: Fetches a random comic from XKCD and displays it in the terminal. Gleeter will automatically skip the comics using an unsupported format (very old comics were using the JPG extension), so you should always get a valid comic. This is a technical limitation of the Terminal Graphics Protocol.
 *   `id <number>`: Fetches the comic with the specified ID and displays it in the terminal. Replace `<number>` with the desired comic ID.
 *   `serve <port> <base_path>`: Starts a web server to serve the comics. `<port>` is optional and defaults to 8080. `<base_path>` is also optional and defaults to "". For example, `gleeter serve 3000 /comics` will start the server on port 3000 and serve the comics under the `/comics` path.
 
@@ -190,6 +190,8 @@ Example response:
 
 This endpoint can be useful for monitoring the health and performance of the server.
 
+The automatic skipping of unsupported images in random mode also applies when using the `/random` endpoint in serve mode, ensuring that only valid comics are served.
+
 #### Prometheus Metrics
 
 Gleeter exposes Prometheus metrics that can be used to monitor the server's performance and health. These metrics are available at the `/<base_path>/metrics` and **optionally** also at the `/metrics` path (the behavior is configurable via the configuration file).
@@ -205,6 +207,7 @@ The following metrics are exposed:
 *   `gleeter:cached_elements_total`: A counter of the total number of elements currently stored in the cache.
 *   `gleeter:memory_used_bytes`: A gauge of the memory used by the Erlang VM in bytes, with labels for different memory slices (e.g., "system", "processes", "ets", "code", "binary", "atom").
 *   `gleeter:http_invalid_requests_total`: A counter of the total number of invalid HTTP requests.
+*   `gleeter:jpeg_retried_total`: A counter of the total number of times the server had to retry because it was getting a JPEG image.
 
 These metrics can be used to create dashboards and alerts to monitor the Gleeter server's performance.
 

docker-test/Gleeter-dashboard.json

@@ -140,15 +140,15 @@
       "pluginVersion": "12.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "editorMode": "code",
           "expr": "sum (gleeter:processed_queries_total) by (method)",
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
+          "refId": "A"
         }
       ],
       "title": "Total processed queries",
@@ -201,15 +201,15 @@
       "pluginVersion": "12.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "editorMode": "code",
           "expr": "sum (gleeter:cached_elements_total) by (instance)",
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
+          "refId": "A"
         }
       ],
       "title": "Cached elements",
@@ -377,7 +377,7 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 24,
+        "w": 17,
         "x": 0,
         "y": 17
       },
@@ -398,18 +398,116 @@
       "pluginVersion": "12.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "editorMode": "code",
           "expr": "sum (rate(gleeter:processed_queries_total[10m])) by (method)",
           "legendFormat": "__auto",
           "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Processed queries per second",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 7,
+        "x": 17,
+        "y": 17
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "last"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "gleeter:jpeg_retried_total",
+          "legendFormat": "Retried",
+          "range": true,
           "refId": "A",
           "datasource": {
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           }
         }
       ],
-      "title": "Processed queries per second",
+      "title": "Retried JPEG",
       "type": "timeseries"
     },
     {
@@ -495,15 +593,15 @@
       "pluginVersion": "12.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "editorMode": "code",
           "expr": "sum (rate(gleeter:http_request_duration_seconds_sum[5m]) / rate(gleeter:http_request_duration_seconds_count[5m])) by (method)",
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
+          "refId": "A"
         }
       ],
       "title": "Average request duration",
@@ -591,15 +689,15 @@
       "pluginVersion": "12.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "editorMode": "code",
           "expr": "gleeter:cached_elements_total",
           "legendFormat": "Total cached elements",
           "range": true,
-          "refId": "Total Elements",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
+          "refId": "Total Elements"
         },
         {
           "datasource": {
@@ -646,6 +744,6 @@
   "timezone": "browser",
   "title": "Gleeter",
   "uid": "22949757-c3b8-4009-bc64-5d8ef0ca9e33",
-  "version": 5,
+  "version": 2,
   "weekStart": ""
-}
+}

docker-test/docker-compose.yaml

@@ -38,5 +38,7 @@ services:
       - target: 9999
         published: 8080
         protocol: tcp
+    environment:
+      GLEETER_DEBUG: "1"
     networks: [internal]
     command: ["serve", "9999", "comics"]

src/gleeter.gleam

@@ -5,13 +5,15 @@ import gleam/io
 import gleam/list
 import gleam/option
 import gleam/result
+import gleam/uri
 import gleeter/application_behavior
 import gleeter/cache
 import gleeter/config
 import gleeter/debug.{debug_print}
 import gleeter/graphics
 import gleeter/png
 import gleeter/serve
+import gleeter/utils
 import gleeter/version
 import gleeter/xkcd
 
@@ -36,7 +38,10 @@ fn print_version() -> Result(Nil, Nil) {
 }
 
 // Get a comic from the cache or fetch it from the APIs and then store in the cache
-fn get_comic(cache: cache.Cache, id: Int) -> Result(cache.ComicWithData, Nil) {
+fn get_comic_from_cache(
+  cache: cache.Cache,
+  id: Int,
+) -> Result(cache.ComicWithData, Nil) {
   case cache.get_comic(cache, id) {
     option.Some(cd) -> Ok(cd)
     option.None -> {
@@ -71,26 +76,39 @@ fn get_comic(cache: cache.Cache, id: Int) -> Result(cache.ComicWithData, Nil) {
   }
 }
 
-fn print_comic(
-  cache: cache.Cache,
-  config: config.Configuration,
+fn get_comic(
   in: PrintComic,
-) -> Result(Nil, Nil) {
-  use cached_comic <- result.try(case in {
-    Latest -> get_comic(cache, 0)
+  cache: cache.Cache,
+) -> Result(cache.ComicWithData, Nil) {
+  case in {
+    Latest -> get_comic_from_cache(cache, 0)
+    ID(id) -> get_comic_from_cache(cache, id)
     Random -> {
       use xkcd.Xkcd(number: highest, ..) <- result.try(
         xkcd.get_latest() |> result.map_error(print_api_error),
       )
-      let random_comic = case config.random_start {
-        option.Some(start) -> int.random(highest - start) + start
-        option.None -> int.random(highest)
+      let random_comic = int.random(highest)
+      use cache.ComicWithData(xkcd.Xkcd(img_url:, ..), ..) as r <- result.try(
+        get_comic_from_cache(cache, random_comic),
+      )
+      let uri_string = uri.to_string(img_url)
+      case utils.is_jpeg(uri_string) {
+        False -> Ok(r)
+        True -> {
+          debug_print("Skipping JPEG comic: " <> uri_string)
+          get_comic(in, cache)
+        }
       }
-      get_comic(cache, random_comic)
     }
-    ID(id) -> get_comic(cache, id)
-  })
+  }
+}
 
+fn print_comic(
+  cache: cache.Cache,
+  config: config.Configuration,
+  in: PrintComic,
+) -> Result(Nil, Nil) {
+  use cached_comic <- result.try(get_comic(in, cache))
   let cache.ComicWithData(xkcd, body, raw_data) = cached_comic
   use image_size <- result.try(png.get_image_size(raw_data))
   let terminal_size = graphics.get_terminal_size()

src/gleeter/metrics.gleam

@@ -24,6 +24,8 @@ const memory_used = "gleeter:memory_used_bytes"
 
 const invalid_requests = "gleeter:http_invalid_requests_total"
 
+const jpeg_retried = "gleeter:jpeg_retried_total"
+
 type Memory {
   Processes(m: Int)
   ProcessesUsed(m: Int)
@@ -52,6 +54,7 @@ pub fn init() -> Nil {
     counter.new(invalid_requests, "Number of invalid HTTP requests")
   let assert Ok(_) =
     gauge.new(memory_used, "Memory used in bytes by the Erlang VM")
+  let assert Ok(_) = counter.new(jpeg_retried, "Number of JPEG retries")
 
   let buckets =
     set.from_list([
@@ -115,6 +118,11 @@ pub fn increment_cache_miss(method: String) -> Nil {
   Nil
 }
 
+pub fn increment_jpeg_retries() -> Nil {
+  let assert Ok(_) = counter.increment(jpeg_retried, dict.new())
+  Nil
+}
+
 pub fn update_memory() -> Nil {
   list.each(memory(), fn(m) {
     let #(label, memory) = case m {