← Commit history

v0.3.5: paginate the wiki component list (the API caps each response at 200 rows; the wiki now has 640 components, so a single fetch dropped the boards) — restores board discovery

barrett-land ·0a14b3c0d5 ·3mo ago ·parent b409aef
1 file changed +39−20
src/main.rs+39−20
@@ -342,27 +342,46 @@ fn up(slug: &str) -> String { /// List component pages from the wiki API (fast + reliable; `adompkg` is dead). fn list_components() -> Value {     let mut items = Vec::new();-    if let Some(body) = curl(&format!("{WIKI}/api/v1/pages?type=component&limit=300")) {-        if let Ok(v) = serde_json::from_slice::<Value>(&body) {-            if let Some(pages) = v.get("pages").and_then(|p| p.as_array()) {-                for pg in pages {-                    let slug = pg.get("slug").and_then(|s| s.as_str()).unwrap_or("").to_string();-                    if slug.is_empty() {-                        continue;-                    }-                    let title = pg.get("title").and_then(|s| s.as_str()).unwrap_or(&slug).to_string();-                    let brief = pg.get("brief").and_then(|s| s.as_str()).unwrap_or("").to_string();-                    // tags may arrive as a JSON-string or a real array-                    let tags = pg.get("tags").map(|t| match t.as_str() {-                        Some(s) => serde_json::from_str::<Value>(s).unwrap_or_else(|_| json!([])),-                        None => t.clone(),-                    }).unwrap_or_else(|| json!([]));-                    let owner = pg.get("author_name").and_then(|s| s.as_str()).unwrap_or("").to_string();-                    let org = pg.get("org_name").and_then(|s| s.as_str()).unwrap_or("").to_string();-                    let is_starred = pg.get("starred").and_then(|s| s.as_bool()).unwrap_or(false);-                    items.push(json!({"slug": slug, "title": title, "brief": brief, "tags": tags, "owner": owner, "org": org, "starred": is_starred}));-                }+    // The wiki caps each list response at ~200 rows regardless of `limit`, and there+    // are now hundreds of components — a single fetch drops most of them (boards+    // included). Page through with `offset` until we've collected every component.+    let mut offset = 0usize;+    loop {+        let body = match curl(&format!("{WIKI}/api/v1/pages?type=component&limit=200&offset={offset}")) {+            Some(b) => b,+            None => break,+        };+        let v = match serde_json::from_slice::<Value>(&body) {+            Ok(v) => v,+            Err(_) => break,+        };+        let pages = match v.get("pages").and_then(|p| p.as_array()) {+            Some(p) if !p.is_empty() => p.clone(),+            _ => break,+        };+        let n = pages.len();+        for pg in &pages {+            let slug = pg.get("slug").and_then(|s| s.as_str()).unwrap_or("").to_string();+            if slug.is_empty() {+                continue;             }+            let title = pg.get("title").and_then(|s| s.as_str()).unwrap_or(&slug).to_string();+            let brief = pg.get("brief").and_then(|s| s.as_str()).unwrap_or("").to_string();+            // tags may arrive as a JSON-string or a real array+            let tags = pg.get("tags").map(|t| match t.as_str() {+                Some(s) => serde_json::from_str::<Value>(s).unwrap_or_else(|_| json!([])),+                None => t.clone(),+            }).unwrap_or_else(|| json!([]));+            let owner = pg.get("author_name").and_then(|s| s.as_str()).unwrap_or("").to_string();+            let org = pg.get("org_name").and_then(|s| s.as_str()).unwrap_or("").to_string();+            let is_starred = pg.get("starred").and_then(|s| s.as_bool()).unwrap_or(false);+            items.push(json!({"slug": slug, "title": title, "brief": brief, "tags": tags, "owner": owner, "org": org, "starred": is_starred}));+        }+        offset += n;+        let total = v.get("total").and_then(|t| t.as_u64()).unwrap_or(0) as usize;+        // stop when we've read everything, the page came back short, or as a hard cap+        if offset >= total || n < 200 || offset >= 5000 {+            break;         }     }     if items.is_empty() {