1 #lang typed/racket/no-check
10 (require (prefix-in info: "info.rkt"))
18 (define-type Out-Format
22 (define-type Timeline-Order
39 (define (concurrent-filter-map num-workers f xs)
40 ; TODO preserve order of elements OR communicate that reorder is expected
41 ; TODO switch from mailboxes to channels
42 (define (make-worker id f)
43 (define parent (current-thread))
45 (define self (current-thread))
47 (thread-send parent (cons 'next self))
48 (match (thread-receive)
49 ['done (thread-send parent (cons 'exit id))]
50 [(cons 'unit x) (begin
52 (when y (thread-send parent (cons 'result y)))
55 (define (dispatch ws xs ys)
58 (match (thread-receive)
59 [(cons 'exit w) (dispatch (remove w ws =) xs ys)]
60 [(cons 'result y) (dispatch ws xs (cons y ys))]
61 [(cons 'next thd) (match xs
63 (thread-send thd 'done)
66 (thread-send thd (cons 'unit x))
67 (dispatch ws xs ys))])])))
68 (define workers (range num-workers))
69 (define threads (map (λ (id) (thread (make-worker id f))) workers))
70 (define results (dispatch workers xs '()))
71 (for-each thread-wait threads)
75 (let* ([f (λ (x) (if (even? x) x #f))]
77 [actual (sort (concurrent-filter-map 10 f xs) <)]
78 [expected (sort ( filter-map f xs) <)])
79 (check-equal? actual expected "concurrent-filter-map")))
81 (: msg-print (-> Out-Format Integer Msg Void))
83 (let* ([colors (vector 36 33)]
84 [n (vector-length colors)])
85 (λ (out-format color-i msg)
86 (let ([color (vector-ref colors (modulo color-i n))]
88 [uri (url->string (msg-uri msg))]
89 [text (msg-text msg)])
92 (printf "~a \033[1;37m<~a>\033[0m \033[0;~am~a\033[0m~n"
93 (parameterize ([date-display-format 'iso-8601])
94 (date->string (seconds->date [msg-ts-epoch msg]) #t))
97 (printf "~a (~a)~n\033[1;37m<~a ~a>\033[0m~n\033[0;~am~a\033[0m~n~n"
98 (parameterize ([date-display-format 'rfc2822])
99 (date->string (seconds->date [msg-ts-epoch msg]) #t))
101 nick uri color text)])))))
103 (: rfc3339->epoch (-> String (Option Nonnegative-Integer)))
104 (define rfc3339->epoch
105 (let ([re (pregexp "^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(:([0-9]{2}))?(\\.[0-9]+)?(Z|([+-])([0-9]{1,2}):?([0-9]{2}))?$")])
107 (match (regexp-match re ts)
108 [(list _wholething yyyy mm dd HH MM _:SS SS _fractional tz-whole tz-sign tz-HH tz-MM)
111 (match* (tz-whole tz-sign tz-HH tz-MM)
114 [(_ (or "-" "+") (? identity h) (? identity m))
115 (let ([h (string->number h)]
116 [m (string->number m)]
117 ; Reverse to get back to UTC:
118 [op (match tz-sign ["+" -] ["-" +])])
119 (op 0 (+ (* 60 m) (* 60 (* 60 h)))))]
121 (log-warning "Impossible TZ string: ~v, components: ~v ~v ~v ~v" tz-whole a b c d)
125 [ts-epoch (find-seconds (if SS (string->number SS) 0)
130 (string->number yyyy)
132 (+ ts-epoch tz-offset))]
134 (log-error "Invalid timestamp: ~v" ts)
137 (: str->msg (-> String Url String (Option Msg)))
139 (let ([re (pregexp "^([^\\s\t]+)[\\s\t]+(.*)$")])
145 "Failed to parse msg: ~v, from: ~v, at: ~v, because: ~v"
146 str nick (url->string uri) e)
148 (match (regexp-match re str)
149 [(list _wholething ts-orig text)
150 (let ([ts-epoch (rfc3339->epoch ts-orig)])
152 (msg ts-epoch ts-orig nick uri text)
155 "Msg rejected due to invalid timestamp: ~v, nick:~v, uri:~v"
156 str nick (url->string uri))
159 (log-debug "Non-msg line from nick:~a, line:~a" nick str)
163 (let* ([tzs (for*/list ([d '("-" "+")]
165 [m '("00" ":00" "57" ":57")])
166 (string-append d h m))]
167 [tzs (list* "" "Z" tzs)])
168 (for* ([n '("fake-nick")]
173 [sep (list "\t" " ")]
174 [txt '("foo bar baz" "'jaz poop bear giraffe / tea" "@*\"``")])
175 (let* ([ts (string-append "2020-11-18T22:22"
176 (if (non-empty-string? s) s ":00")
178 [m (str->msg n u (string-append ts sep txt))])
180 (check-equal? (msg-nick m) n)
181 (check-equal? (msg-uri m) u)
182 (check-equal? (msg-text m) txt)
183 (check-equal? (msg-ts-orig m) ts (format "Given: ~v" ts))
186 (let* ([ts "2020-11-18T22:22:09-0500"]
191 [actual (str->msg nick uri (string-append ts tab text))]
192 [expected (msg 1605756129 ts nick uri text)])
194 (msg-ts-epoch actual)
195 (msg-ts-epoch expected)
199 (msg-ts-orig expected)
214 (: str->lines (-> String (Listof String)))
215 (define (str->lines str)
216 (string-split str (regexp "[\r\n]+")))
219 (check-equal? (str->lines "abc\ndef\n\nghi") '("abc" "def" "ghi")))
221 (: str->msgs (-> String Url String (Listof Msg)))
222 (define (str->msgs nick uri str)
223 (filter-map (λ (line) (str->msg nick uri line)) (filter-comments (str->lines str))))
225 (: hash-sha1 (-> String String))
226 (define (hash-sha1 str)
227 (define in (open-input-string str))
228 (define digest (sha1 in))
229 (close-input-port in)
232 (: url->cache-file-path (-> Url Path-String))
233 (define (url->cache-file-path uri)
234 ; TODO Replace hashing with encoding
235 (expand-user-path (string-append "~/.tt/cache/" (hash-sha1 (url->string uri)))))
238 (: uri-read-cached (-> Url String))
239 (define (uri-read-cached uri)
240 (define path (url->cache-file-path uri))
241 (if (file-exists? path)
244 (log-warning "Cache file not found for URI: ~a" (url->string uri))
247 (: str->feed (String (Option Feed)))
248 (define (str->feed str)
249 (log-debug "Parsing feed string: ~v" str)
250 (match (string-split str)
255 (log-error "Invalid URI: ~v, exn: ~v" u e)
257 (feed nick (string->url u)))]
259 (log-error "Invalid feed string: ~v" str)
262 (: filter-comments (-> (Listof String) (Listof String)))
263 (define (filter-comments lines)
264 (filter-not (λ (line) (string-prefix? line "#")) lines))
266 (: str->feeds (-> String (Listof Feed)))
267 (define (str->feeds str)
268 (filter-map str->feed (filter-comments (str->lines str))))
270 (: file->feeds (-> Path-String (Listof Feed)))
271 (define (file->feeds filename)
272 (str->feeds (file->string filename)))
274 (: user-agent String)
278 [prog-version (info:#%info-lookup 'version)]
279 [prog-uri "https://github.com/xandkar/tt"]
280 [user-feed-file (expand-user-path "~/twtxt-me.txt")]
282 (if (file-exists? user-feed-file)
283 (let ([user (first (file->feeds user-feed-file))])
284 (format "+~a; @~a" (url->string (feed-uri user)) (feed-nick user)))
285 (format "+~a" prog-uri))])
286 (format "~a/~a (~a)" prog-name prog-version user)))
288 (: uri-download (-> Url Void))
289 (define (uri-download u)
290 (define cache-file-path (url->cache-file-path u))
291 (log-debug "uri-download ~v into ~v" u cache-file-path)
292 (match* ((url-scheme u) (url-host u) (url-port u))
295 (define ssl? (string=? s "https"))
296 (define-values (status-line headers body-input)
297 ; TODO Timeout. Currently hangs on slow connections.
300 (url->string (struct-copy url u [scheme #f] [host #f]))
302 #:port (cond [p p] [ssl? 443] [else 80])
303 #:headers (list (format "User-Agent: ~a" user-agent))
305 (log-debug "headers: ~v" headers)
306 (log-debug "status-line: ~v" status-line)
308 (string->number (second (string-split (bytes->string/utf-8 status-line)))))
309 (log-debug "status: ~v" status)
310 ; TODO Handle redirects
312 (call-with-output-file cache-file-path
314 (copy-port body-input cache-output))
318 (log-error "Invalid URI: ~v" u)]))
320 (: timeline-print (-> Out-Format (Listof Msg) Void))
321 (define (timeline-print out-format timeline)
322 (void (foldl (match-lambda**
323 [((and m (msg _ _ nick _ _)) (cons prev-nick i))
324 (let ([i (if (string=? prev-nick nick) i (+ 1 i))])
325 (msg-print out-format i m)
330 (: feed->msgs (-> Feed (Listof Msg)))
331 (define (feed->msgs f)
332 (match-define (feed nick uri) f)
333 (log-info "Reading feed nick:~a uri:~v" nick (url->string uri))
334 (str->msgs nick uri (uri-read-cached uri)))
336 (: feed-download (-> Feed Void))
337 (define (feed-download f)
338 (match-define (feed nick uri) f)
339 (log-info "Downloading feed nick:~a uri:~a" nick (url->string uri))
343 (log-error "Network error nick:~a uri:~v exn:~v" nick uri e)
347 (log-error "HTTP error nick:~a uri:~a status:~a" nick uri status)
351 (: timeline-download (-> Integer (Listof Feed) Void))
352 (define (timeline-download num-workers feeds)
353 ; TODO No need for map - can just iter
354 (void (concurrent-filter-map num-workers feed-download feeds)))
356 ; TODO timeline contract : time-sorted list of messages
357 (: timeline-read (-> Timeline-Order (Listof Feed) (Listof Msg)))
358 (define (timeline-read order feeds)
359 (define cmp (match order
362 (sort (append* (filter-map feed->msgs feeds))
363 (λ (a b) (cmp (msg-ts-epoch a) (msg-ts-epoch b)))))
365 (: start-logger (-> Log-Level Void))
366 (define (start-logger level)
367 (let* ([logger (make-logger #f #f level #f)]
368 [log-receiver (make-log-receiver logger level)])
371 ([date-display-format 'iso-8601])
373 (define data (sync log-receiver))
374 (define level (vector-ref data 0))
375 (define msg (vector-ref data 1))
376 (define ts (date->string (current-date) #t))
377 (eprintf "~a [~a] ~a~n" ts level msg)
379 (current-logger logger)))
382 (let ([log-level 'info])
388 "Enable debug log level."
389 (set! log-level 'debug)]
392 "and <command> is one of"
393 "r, read i : Read the timeline."
394 "d, download : Download the timeline."
395 "u, upload : Upload your twtxt file (alias to execute ~/.tt/upload)."
397 #:args (command . args)
398 (start-logger log-level)
399 (current-command-line-arguments (list->vector args))
402 (let ([num-workers 15]) ; 15 was fastest out of the tried: 1, 5, 10, 20.
408 njobs "Number of concurrent jobs."
409 (set! num-workers (string->number njobs))]
411 (timeline-download num-workers (file->feeds filename))))]
417 (if (system (path->string (expand-user-path "~/.tt/upload")))
421 (let ([out-format 'multi-line]
428 "Reverse displayed timeline order."
429 (set! order 'new->old)]
432 "Short output format"
433 (set! out-format 'single-line)]
436 (set! out-format 'multi-line)]
438 (timeline-print out-format (timeline-read order (file->feeds filename)))))]