1 #lang typed/racket/no-check
10 (require (prefix-in info: "info.rkt"))
18 (define-type Out-Format
22 (define-type Timeline-Order
39 (: concurrent-filter-map (∀ (α β) (-> Natural (-> α β) (Listof α))))
40 (define (concurrent-filter-map num-workers f xs)
41 ; TODO preserve order of elements OR communicate that reorder is expected
42 ; TODO switch from mailboxes to channels
43 (define (make-worker id f)
44 (define parent (current-thread))
46 (define self : Thread (current-thread))
47 (: work (∀ (α) (-> α)))
49 (thread-send parent (cons 'next self))
50 (match (thread-receive)
51 ['done (thread-send parent (cons 'exit id))]
52 [(cons 'unit x) (begin
54 (when y (thread-send parent (cons 'result y)))
57 (: dispatch (∀ (α β) (-> (Listof Nonnegative-Integer) (Listof α) (Listof β))))
58 (define (dispatch ws xs ys)
61 (match (thread-receive)
62 [(cons 'exit w) (dispatch (remove w ws =) xs ys)]
63 [(cons 'result y) (dispatch ws xs (cons y ys))]
64 [(cons 'next thd) (match xs
66 (thread-send thd 'done)
69 (thread-send thd (cons 'unit x))
70 (dispatch ws xs ys))])])))
71 (define workers (range num-workers))
72 (define threads (map (λ (id) (thread (make-worker id f))) workers))
73 (define results (dispatch workers xs '()))
74 (for-each thread-wait threads)
78 (let* ([f (λ (x) (if (even? x) x #f))]
80 [actual (sort (concurrent-filter-map 10 f xs) <)]
81 [expected (sort ( filter-map f xs) <)])
82 (check-equal? actual expected "concurrent-filter-map")))
84 (: msg-print (-> Out-Format Integer Msg Void))
86 (let* ([colors (vector 36 33)]
87 [n (vector-length colors)])
88 (λ (out-format color-i msg)
89 (let ([color (vector-ref colors (modulo color-i n))]
91 [uri (url->string (msg-uri msg))]
92 [text (msg-text msg)])
95 (printf "~a \033[1;37m<~a>\033[0m \033[0;~am~a\033[0m~n"
96 (parameterize ([date-display-format 'iso-8601])
97 (date->string (seconds->date [msg-ts-epoch msg]) #t))
100 (printf "~a (~a)~n\033[1;37m<~a ~a>\033[0m~n\033[0;~am~a\033[0m~n~n"
101 (parameterize ([date-display-format 'rfc2822])
102 (date->string (seconds->date [msg-ts-epoch msg]) #t))
104 nick uri color text)])))))
106 (: rfc3339->epoch (-> String (Option Nonnegative-Integer)))
107 (define rfc3339->epoch
108 (let ([re (pregexp "^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(:([0-9]{2}))?(\\.[0-9]+)?(Z|([+-])([0-9]{1,2}):?([0-9]{2}))?$")])
110 (match (regexp-match re ts)
111 [(list _wholething yyyy mm dd HH MM _:SS SS _fractional tz-whole tz-sign tz-HH tz-MM)
114 (match* (tz-whole tz-sign tz-HH tz-MM)
117 [(_ (or "-" "+") (? identity h) (? identity m))
118 (let ([h (string->number h)]
119 [m (string->number m)]
120 ; Reverse to get back to UTC:
121 [op (match tz-sign ["+" -] ["-" +])])
122 (op 0 (+ (* 60 m) (* 60 (* 60 h)))))]
124 (log-warning "Impossible TZ string: ~v, components: ~v ~v ~v ~v" tz-whole a b c d)
128 [ts-epoch (find-seconds (if SS (string->number SS) 0)
133 (string->number yyyy)
135 (+ ts-epoch tz-offset))]
137 (log-error "Invalid timestamp: ~v" ts)
140 (: str->msg (-> String Url String (Option Msg)))
142 (let ([re (pregexp "^([^\\s\t]+)[\\s\t]+(.*)$")])
148 "Failed to parse msg: ~v, from: ~v, at: ~v, because: ~v"
149 str nick (url->string uri) e)
151 (match (regexp-match re str)
152 [(list _wholething ts-orig text)
153 (let ([ts-epoch (rfc3339->epoch ts-orig)])
155 (msg ts-epoch ts-orig nick uri text)
158 "Msg rejected due to invalid timestamp: ~v, nick:~v, uri:~v"
159 str nick (url->string uri))
162 (log-debug "Non-msg line from nick:~a, line:~a" nick str)
166 (let* ([tzs (for*/list ([d '("-" "+")]
168 [m '("00" ":00" "57" ":57")])
169 (string-append d h m))]
170 [tzs (list* "" "Z" tzs)])
171 (for* ([n '("fake-nick")]
176 [sep (list "\t" " ")]
177 [txt '("foo bar baz" "'jaz poop bear giraffe / tea" "@*\"``")])
178 (let* ([ts (string-append "2020-11-18T22:22"
179 (if (non-empty-string? s) s ":00")
181 [m (str->msg n u (string-append ts sep txt))])
183 (check-equal? (msg-nick m) n)
184 (check-equal? (msg-uri m) u)
185 (check-equal? (msg-text m) txt)
186 (check-equal? (msg-ts-orig m) ts (format "Given: ~v" ts))
189 (let* ([ts "2020-11-18T22:22:09-0500"]
194 [actual (str->msg nick uri (string-append ts tab text))]
195 [expected (msg 1605756129 ts nick uri text)])
197 (msg-ts-epoch actual)
198 (msg-ts-epoch expected)
202 (msg-ts-orig expected)
217 (: str->lines (-> String (Listof String)))
218 (define (str->lines str)
219 (string-split str (regexp "[\r\n]+")))
222 (check-equal? (str->lines "abc\ndef\n\nghi") '("abc" "def" "ghi")))
224 (: str->msgs (-> String Url String (Listof Msg)))
225 (define (str->msgs nick uri str)
226 (filter-map (λ (line) (str->msg nick uri line)) (filter-comments (str->lines str))))
228 (: hash-sha1 (-> String String))
229 (define (hash-sha1 str)
230 (define in (open-input-string str))
231 (define digest (sha1 in))
232 (close-input-port in)
235 (: url->cache-file-path (-> Url Path-String))
236 (define (url->cache-file-path uri)
237 ; TODO Replace hashing with encoding
238 (expand-user-path (string-append "~/.tt/cache/" (hash-sha1 (url->string uri)))))
241 (: uri-read-cached (-> Url String))
242 (define (uri-read-cached uri)
243 (define path (url->cache-file-path uri))
244 (if (file-exists? path)
247 (log-warning "Cache file not found for URI: ~a" (url->string uri))
250 (: str->feed (String (Option Feed)))
251 (define (str->feed str)
252 (log-debug "Parsing feed string: ~v" str)
253 (match (string-split str)
258 (log-error "Invalid URI: ~v, exn: ~v" u e)
260 (feed nick (string->url u)))]
262 (log-error "Invalid feed string: ~v" str)
265 (: filter-comments (-> (Listof String) (Listof String)))
266 (define (filter-comments lines)
267 (filter-not (λ (line) (string-prefix? line "#")) lines))
269 (: str->feeds (-> String (Listof Feed)))
270 (define (str->feeds str)
271 (filter-map str->feed (filter-comments (str->lines str))))
273 (: file->feeds (-> Path-String (Listof Feed)))
274 (define (file->feeds filename)
275 (str->feeds (file->string filename)))
277 (: user-agent String)
281 [prog-version (info:#%info-lookup 'version)]
282 [prog-uri "https://github.com/xandkar/tt"]
283 [user-feed-file (expand-user-path "~/twtxt-me.txt")]
285 (if (file-exists? user-feed-file)
286 (let ([user (first (file->feeds user-feed-file))])
287 (format "+~a; @~a" (url->string (feed-uri user)) (feed-nick user)))
288 (format "+~a" prog-uri))])
289 (format "~a/~a (~a)" prog-name prog-version user)))
291 (: uri-download (-> Url Void))
292 (define (uri-download u)
293 (define cache-file-path (url->cache-file-path u))
294 (log-debug "uri-download ~v into ~v" u cache-file-path)
295 (match* ((url-scheme u) (url-host u) (url-port u))
298 (define ssl? (string=? s "https"))
299 (define-values (status-line headers body-input)
300 ; TODO Timeout. Currently hangs on slow connections.
303 (url->string (struct-copy url u [scheme #f] [host #f]))
305 #:port (cond [p p] [ssl? 443] [else 80])
306 #:headers (list (format "User-Agent: ~a" user-agent))
308 (log-debug "headers: ~v" headers)
309 (log-debug "status-line: ~v" status-line)
311 (string->number (second (string-split (bytes->string/utf-8 status-line)))))
312 (log-debug "status: ~v" status)
313 ; TODO Handle redirects
315 (call-with-output-file cache-file-path
317 (copy-port body-input cache-output))
321 (log-error "Invalid URI: ~v" u)]))
323 (: timeline-print (-> Out-Format (Listof Msg) Void))
324 (define (timeline-print out-format timeline)
325 (void (foldl (match-lambda**
326 [((and m (msg _ _ nick _ _)) (cons prev-nick i))
327 (let ([i (if (string=? prev-nick nick) i (+ 1 i))])
328 (msg-print out-format i m)
333 (: feed->msgs (-> Feed (Listof Msg)))
334 (define (feed->msgs f)
335 (match-define (feed nick uri) f)
336 (log-info "Reading feed nick:~a uri:~v" nick (url->string uri))
337 (str->msgs nick uri (uri-read-cached uri)))
339 (: feed-download (-> Feed Void))
340 (define (feed-download f)
341 (match-define (feed nick uri) f)
342 (log-info "Downloading feed nick:~a uri:~a" nick (url->string uri))
346 (log-error "Network error nick:~a uri:~v exn:~v" nick uri e)
350 (log-error "HTTP error nick:~a uri:~a status:~a" nick uri status)
354 (: timeline-download (-> Integer (Listof Feed) Void))
355 (define (timeline-download num-workers feeds)
356 ; TODO No need for map - can just iter
357 (void (concurrent-filter-map num-workers feed-download feeds)))
359 ; TODO timeline contract : time-sorted list of messages
360 (: timeline-read (-> Timeline-Order (Listof Feed) (Listof Msg)))
361 (define (timeline-read order feeds)
362 (define cmp (match order
365 (sort (append* (filter-map feed->msgs feeds))
366 (λ (a b) (cmp (msg-ts-epoch a) (msg-ts-epoch b)))))
368 (: start-logger (-> Log-Level Void))
369 (define (start-logger level)
370 (let* ([logger (make-logger #f #f level #f)]
371 [log-receiver (make-log-receiver logger level)])
374 ([date-display-format 'iso-8601])
376 (define data (sync log-receiver))
377 (define level (vector-ref data 0))
378 (define msg (vector-ref data 1))
379 (define ts (date->string (current-date) #t))
380 (eprintf "~a [~a] ~a~n" ts level msg)
382 (current-logger logger)))
385 (let ([log-level 'info])
391 "Enable debug log level."
392 (set! log-level 'debug)]
395 "and <command> is one of"
396 "r, read i : Read the timeline."
397 "d, download : Download the timeline."
398 "u, upload : Upload your twtxt file (alias to execute ~/.tt/upload)."
400 #:args (command . args)
401 (start-logger log-level)
402 (current-command-line-arguments (list->vector args))
405 (let ([num-workers 15]) ; 15 was fastest out of the tried: 1, 5, 10, 20.
411 njobs "Number of concurrent jobs."
412 (set! num-workers (string->number njobs))]
414 (timeline-download num-workers (file->feeds filename))))]
420 (if (system (path->string (expand-user-path "~/.tt/upload")))
424 (let ([out-format 'multi-line]
431 "Reverse displayed timeline order."
432 (set! order 'new->old)]
435 "Short output format"
436 (set! out-format 'single-line)]
439 (set! out-format 'multi-line)]
441 (timeline-print out-format (timeline-read order (file->feeds filename)))))]