Validate crawled peers

[tt.git] / tt.rkt

diff --git a/tt.rkt b/tt.rkt
index 5b10215..a375c69 100644 (file)
--- a/tt.rkt
+++ b/tt.rkt
@@ -282,8 +282,7 @@
 (define (url->cache-lmod-path uri)
   (build-path cache-dir "lmods" (uri-encode (url->string uri))))
 
-; TODO Return Option
-(: uri-read-cached (-> Url String))
+(: uri-read-cached (-> Url (Option String)))
 (define (uri-read-cached uri)
   (define path-v1 (url->cache-file-path-v1 uri))
   (define path-v2 (url->cache-file-path-v2 uri))
@@ -293,7 +292,7 @@
       (file->string path-v2)
       (begin
         (log-warning "Cache file not found for URI: ~a" (url->string uri))
-        "")))
+        #f)))
 
 (: str->url (-> String (Option String)))
 (define (str->url s)
@@ -566,7 +565,10 @@
 (define (peer->msgs peer)
   (match-define (Peer nick uri _) peer)
   (log-info "Reading peer nick:~v uri:~v" nick (url->string uri))
-  (str->msgs nick uri (uri-read-cached uri)))
+  (define msgs-data (uri-read-cached uri))
+  (if msgs-data
+      (str->msgs nick uri msgs-data)
+      '()))
 
 (: peer-download
    (-> Positive-Float Peer
@@ -793,6 +795,8 @@
                     (build-path tt-home-dir "peers-all")]
                   [peers-mentioned-file
                     (build-path tt-home-dir "peers-mentioned")]
+                  [peers-parsed-file
+                    (build-path tt-home-dir "peers-parsed")]
                   [peers-mentioned-curr
                     (mentioned-peers-in-cache)]
                   [peers-mentioned-prev
@@ -808,8 +812,13 @@
                   [peers-discovered
                     (set-subtract peers-all (list->set peers-all-prev))]
                   [peers-all
-                    (peers-sort (set->list peers-all))])
+                    (peers-sort (set->list peers-all))]
+                  [peers-parsed
+                    (filter
+                      (λ (p) (< 0 (length (peer->msgs p))))
+                      peers-all)])
              (log-info "Known peers mentioned: ~a" (length peers-mentioned))
+             (log-info "Known peers parsed ~a" (length peers-parsed))
              (log-info "Known peers total: ~a" (length peers-all))
              (log-info "Discovered ~a new peers:~n~a"
                        (set-count peers-discovered)
@@ -819,6 +828,8 @@
                                         (set->list peers-discovered))))
              (peers->file peers-mentioned
                           peers-mentioned-file)
+             (peers->file peers-parsed
+                          peers-parsed-file)
              (peers->file peers-all
                           peers-all-file)))]
         [command