From: Siraaj Khandkar <siraaj@khandkar.net>
Date: Fri, 26 Nov 2021 20:21:51 +0000 (-0500)
Subject: Validate crawled peers
X-Git-Tag: 0.26.0
X-Git-Url: https://git.xandkar.net/?p=tt.git;a=commitdiff_plain;h=b39ba7bee95268e33eddab23bf19583a66f8c123

Validate crawled peers

if at least one message could be parsed from downloaded timeline.
---

diff --git a/info.rkt b/info.rkt
index 22c222b..24b34a3 100644
--- a/info.rkt
+++ b/info.rkt
@@ -6,7 +6,7 @@
 (define pkg-desc
   "twtxt client")
 (define version
-  "0.25.0")
+  "0.26.0")
 (define pkg-authors
   '("Siraaj Khandkar <siraaj@khandkar.net>"))
 (define deps
diff --git a/tt.rkt b/tt.rkt
index 2fbcfe2..a375c69 100644
--- a/tt.rkt
+++ b/tt.rkt
@@ -795,6 +795,8 @@
                     (build-path tt-home-dir "peers-all")]
                   [peers-mentioned-file
                     (build-path tt-home-dir "peers-mentioned")]
+                  [peers-parsed-file
+                    (build-path tt-home-dir "peers-parsed")]
                   [peers-mentioned-curr
                     (mentioned-peers-in-cache)]
                   [peers-mentioned-prev
@@ -810,8 +812,13 @@
                   [peers-discovered
                     (set-subtract peers-all (list->set peers-all-prev))]
                   [peers-all
-                    (peers-sort (set->list peers-all))])
+                    (peers-sort (set->list peers-all))]
+                  [peers-parsed
+                    (filter
+                      (λ (p) (< 0 (length (peer->msgs p))))
+                      peers-all)])
              (log-info "Known peers mentioned: ~a" (length peers-mentioned))
+             (log-info "Known peers parsed ~a" (length peers-parsed))
              (log-info "Known peers total: ~a" (length peers-all))
              (log-info "Discovered ~a new peers:~n~a"
                        (set-count peers-discovered)
@@ -821,6 +828,8 @@
                                         (set->list peers-discovered))))
              (peers->file peers-mentioned
                           peers-mentioned-file)
+             (peers->file peers-parsed
+                          peers-parsed-file)
              (peers->file peers-all
                           peers-all-file)))]
         [command