From e4bfe7299d5a2342fa12442ecdfe06acbeb82649 Mon Sep 17 00:00:00 2001 From: Aafaq-rashid-comprinno Date: Wed, 24 Jun 2026 18:49:04 +0530 Subject: [PATCH] feat(index): default to --wait-network-idle for URL rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most modern pages are JS-rendered SPAs that produce blank or incomplete tiles without waiting for network idle. The pixelbrowse skill already uses this flag, but `pixelrag index build` did not — meaning users building indexes of dynamic sites got empty/broken tiles silently. Now the pipeline defaults wait_network_idle=True. Users can opt out with `ingest: {wait_network_idle: false}` in pixelrag.yaml. Addresses #89. --- index/src/pixelrag_index/pipelines.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py index f943ec8..821422c 100644 --- a/index/src/pixelrag_index/pipelines.py +++ b/index/src/pixelrag_index/pipelines.py @@ -28,6 +28,10 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path: tiles_dir = output / "tiles" embeddings_dir = output / "embeddings" ingest_cfg = config.get("ingest", {}) + # Default to waiting for network idle — most modern pages are JS-rendered + # SPAs that produce blank/incomplete tiles without this. Users can opt out + # with `ingest: {wait_network_idle: false}` in their pixelrag.yaml. + ingest_cfg.setdefault("wait_network_idle", True) embed_cfg = config.get("embed", {}) device = embed_cfg.get("device", "cpu")