+; TODO Track timestamps of when a nick was seen.
+; TODO Track number of times a nick was seen.
+(: peers->nicks-by-url (-> (Listof Peer) (Immutable-HashTable Url (Setof String))))
+(define (peers->nicks-by-url peers)
+ (foldl
+ (match-lambda**
+ [((Peer #f _ _ _) tbl) tbl]
+ [((Peer n u _ _) tbl) (hash-update tbl u (λ (nicks) (set-add nicks n)) (set))])
+ (hash)
+ peers))
+
+(: update-nicks-history (-> (Listof Peer) Void))
+(define (update-nicks-history peers)
+ (hash-for-each
+ (peers->nicks-by-url peers)
+ (λ (url nicks-curr)
+ (define path (build-path tt-home-dir "nicks" "seen" (uri-encode (url->string url))))
+ (define nicks-prev
+ (if (file-exists? path)
+ (list->set (file->lines path))
+ (begin
+ (make-parent-directory* path)
+ (set))))
+ (display-lines-to-file
+ (set->list (set-union nicks-prev nicks-curr))
+ path
+ #:exists 'replace))))
+
+(: crawl (-> Void))
+(define (crawl)
+ ; TODO Test the non-io parts of crawling
+ (let* ([peers-all-file
+ (build-path tt-home-dir "peers-all")]
+ [peers-mentioned-file
+ (build-path tt-home-dir "peers-mentioned")]
+ [peers-parsed-file
+ (build-path tt-home-dir "peers-parsed")]
+ [peers-cached-file
+ (build-path tt-home-dir "peers-cached")]
+ [peers-cached
+ (peers-cached)]
+ [cached-timeline
+ (peers->timeline peers-cached)]
+ [peers-mentioned-curr
+ (peers-mentioned cached-timeline)]
+ [peers-mentioned-prev
+ (file->peers peers-mentioned-file)]
+ [peers-all-prev
+ (file->peers peers-all-file)]
+ [peers-mentioned
+ (begin
+ ; XXX Updating nicks before running peers-merge,
+ ; since peers-merge is expected to refer to it in the future.
+ (update-nicks-history (append peers-cached
+ peers-mentioned-curr
+ peers-mentioned-prev
+ peers-all-prev))
+ (peers-merge peers-mentioned-prev
+ peers-mentioned-curr))]
+ [peers-all
+ (peers-merge peers-mentioned
+ peers-all-prev
+ peers-cached)]
+ [peers-discovered
+ (set->list (set-subtract (make-immutable-peers peers-all)
+ (make-immutable-peers peers-all-prev)))]
+ [peers-parsed
+ (filter (λ (p) (> (length (peer->msgs p)) 0)) peers-all)])
+ ; TODO Deeper de-duping
+ (log-info "Known peers cached ~a" (length peers-cached))
+ (log-info "Known peers mentioned: ~a" (length peers-mentioned))
+ (log-info "Known peers parsed ~a" (length peers-parsed))
+ (log-info "Known peers total: ~a" (length peers-all))
+ (log-info "Discovered ~a new peers:~n~a"
+ (length peers-discovered)
+ (pretty-format (map
+ (match-lambda
+ [(Peer n _ u c) (list n u c)])
+ peers-discovered)))
+ (peers->file peers-cached
+ peers-cached-file)
+ (peers->file peers-mentioned
+ peers-mentioned-file)
+ (peers->file peers-parsed
+ peers-parsed-file)
+ (peers->file peers-all
+ peers-all-file)))
+
+(: read (-> (Listof String) Number Number Timeline-Order Out-Format Void))
+(define (read file-paths ts-min ts-max order out-format)
+ (let* ([peers
+ (paths->peers file-paths)]
+ [msgs
+ (timeline-sort (peers->timeline peers) order)]
+ [include?
+ (λ (m)
+ (and (or (not ts-min) (>= (Msg-ts-epoch m) ts-min))
+ (or (not ts-max) (<= (Msg-ts-epoch m) ts-max))))])
+ (timeline-print out-format (filter include? msgs))))
+
+(: upload (-> Void))
+(define (upload)
+ ; FIXME Should not exit from here, but only after cleanup/logger-stoppage.
+ (if (system (path->string (build-path tt-home-dir "hooks" "upload")))
+ (exit 0)
+ (exit 1)))
+
+(: download (-> (Listof String) Positive-Integer Positive-Float Void))
+(define (download file-paths num-workers timeout)
+ (let ([peers (paths->peers file-paths)])
+ (define-values (_res _cpu real-ms _gc)
+ (time-apply timeline-download (list num-workers timeout peers)))
+ (log-info "Downloaded timelines from ~a peers in ~a seconds."
+ (length peers)
+ (/ real-ms 1000.0))))
+
+(: dispatch (-> String Void))
+(define (dispatch command)
+ (match command
+ [(or "d" "download")
+ ; Initially, 15 was fastest out of the tried: 1, 5, 10, 20. Then I
+ ; started noticing significant slowdowns. Reducing to 5 seems to help.
+ (let ([num-workers 5]
+ [timeout 10.0])
+ (command-line
+ #:program "tt download"
+ #:once-each
+ [("-j" "--jobs")
+ njobs "Number of concurrent jobs."
+ (set! num-workers (string->number njobs))]
+ [("-t" "--timeout")
+ seconds "Timeout seconds per request."
+ (set! timeout (string->number seconds))]
+ #:args file-paths
+ (download file-paths num-workers timeout)))]
+ [(or "u" "upload")
+ (command-line
+ #:program "tt upload" #:args () (upload))]
+ [(or "r" "read")
+ (let ([out-format 'multi-line]
+ [order 'old->new]
+ [ts-min #f]
+ [ts-max #f])
+ (command-line
+ #:program "tt read"
+ #:once-each
+ [("-r" "--rev")
+ "Reverse displayed timeline order."
+ (set! order 'new->old)]
+ [("-m" "--min")
+ m "Earliest time to display (ignore anything before it)."
+ (set! ts-min (rfc3339->epoch m))]
+ [("-x" "--max")
+ x "Latest time to display (ignore anything after it)."
+ (set! ts-max (rfc3339->epoch x))]
+ #:once-any
+ [("-s" "--short")
+ "Short output format"
+ (set! out-format 'single-line)]
+ [("-l" "--long")
+ "Long output format"
+ (set! out-format 'multi-line)]
+ #:args file-paths
+ (read file-paths ts-min ts-max order out-format)))]
+ [(or "c" "crawl")
+ (command-line
+ #:program "tt crawl" #:args () (crawl))]
+ [command
+ (eprintf "Error: invalid command: ~v\n" command)
+ (eprintf "Please use the \"--help\" option to see a list of available commands.\n")
+ (exit 1)]))
+