10 (require (prefix-in info: "info.rkt"))
20 uri ; net/url-structs:url
27 uri ; net/url-structs:url
30 (define (concurrent-filter-map num_workers f xs)
31 ; TODO preserve order of elements OR communicate that reorder is expected
32 ; TODO switch from mailboxes to channels
33 (define (make-worker id f)
34 (define parent (current-thread))
36 (define self (current-thread))
38 (thread-send parent (cons 'next self))
39 (match (thread-receive)
40 ['done (thread-send parent (cons 'exit id))]
41 [(cons 'unit x) (begin
43 (when y (thread-send parent (cons 'result y)))
46 (define (dispatch ws xs ys)
49 (match (thread-receive)
50 [(cons 'exit w) (dispatch (remove w ws =) xs ys)]
51 [(cons 'result y) (dispatch ws xs (cons y ys))]
52 [(cons 'next thd) (match xs
54 (thread-send thd 'done)
57 (thread-send thd (cons 'unit x))
58 (dispatch ws xs ys))])])))
59 (define workers (range num_workers))
60 (define threads (map (λ (id) (thread (make-worker id f))) workers))
61 (define results (dispatch workers xs '()))
62 (for-each thread-wait threads)
66 (let* ([f (λ (x) (if (even? x) x #f))]
68 [actual (sort (concurrent-filter-map 10 f xs) <)]
69 [expected (sort ( filter-map f xs) <)])
70 (check-equal? actual expected "concurrent-filter-map")))
73 (let* ([colors (vector 36 33)]
74 [n (vector-length colors)])
75 (λ (out-format color-i msg)
76 (let ([color (vector-ref colors (modulo color-i n))]
78 [uri (url->string (msg-uri msg))]
79 [text (msg-text msg)])
82 (printf "~a \033[1;37m<~a>\033[0m \033[0;~am~a\033[0m~n"
83 (parameterize ([date-display-format 'iso-8601])
84 (date->string (seconds->date [msg-ts_epoch msg]) #t))
87 (printf "~a~n\033[1;37m<~a ~a>\033[0m~n\033[0;~am~a\033[0m~n~n"
88 (parameterize ([date-display-format 'rfc2822])
89 (date->string (seconds->date [msg-ts_epoch msg]) #t))
90 nick uri color text)])))))
93 ; TODO Split parsing into 2 stages: 1) line->list; 2) rfc3339->epoch.
94 (let ([re (pregexp "^(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(:([0-9]{2}))?)(\\.[0-9]+)?([^\\s\t]*)[\\s\t]+(.*)$")])
100 "Failed to parse msg: ~v, from: ~v, at: ~v, because: ~v"
101 str nick (url->string uri) e)
103 (match (regexp-match re str)
104 [(list _wholething ts yyyy mm dd HH MM _:SS SS _f tz text)
106 ; TODO handle tz offset
107 ([ts_rfc3339 (string-append ts (if SS "" ":00") (if tz tz ""))]
108 [ts_epoch (find-seconds (if SS (string->number SS) 0)
113 (string->number yyyy))])
114 (msg ts_epoch ts_rfc3339 nick uri text))]
116 (log-debug "Non-msg line from nick:~a, line:~a" nick str)
120 (let* ([tzs (for*/list ([d '("-" "+")]
122 [m '("00" ":00" "57" ":57")])
123 (string-append d h m))]
124 [tzs (list* "" "Z" tzs)])
125 (for* ([n '("fake-nick")]
130 [sep (list "\t" " ")]
131 [txt '("foo bar baz" "'jaz poop bear giraffe / tea" "@*\"``")])
132 (let* ([ts (string-append "2020-11-18T22:22"
133 (if (non-empty-string? s) s ":00")
135 [m (str->msg n u (string-append ts sep txt))])
137 (check-equal? (msg-nick m) n)
138 (check-equal? (msg-uri m) u)
139 (check-equal? (msg-text m) txt)
140 (check-equal? (msg-ts_rfc3339 m) ts (format "Given: ~v" ts))
143 (let* ([ts "2020-11-18T22:22:09-0500"]
148 [actual (str->msg nick uri (string-append ts tab text))]
149 [expected (msg 1605756129 ts nick uri text)])
150 ; FIXME re-enable after handling tz offset
152 ; (msg-ts_epoch actual)
153 ; (msg-ts_epoch expected)
154 ; "str->msg ts_epoch")
156 (msg-ts_rfc3339 actual)
157 (msg-ts_rfc3339 expected)
158 "str->msg ts_rfc3339")
172 (define (str->lines str)
173 (string-split str (regexp "[\r\n]+")))
176 (check-equal? (str->lines "abc\ndef\n\nghi") '("abc" "def" "ghi")))
178 (define (str->msgs nick uri str)
179 (filter-map (λ (line) (str->msg nick uri line)) (str->lines str)))
181 (define (hash-sha1 str)
182 (define in (open-input-string str))
183 (define digest (sha1 in))
184 (close-input-port in)
187 (define (url->cache-file-path uri)
188 ; TODO Replace hashing with encoding
189 (expand-user-path (string-append "~/.tt/cache/" (hash-sha1 (url->string uri)))))
191 (define (uri-read-cached uri)
192 (define path (url->cache-file-path uri))
193 (if (file-exists? path)
196 (log-warning "Cache file not found for URI: ~a" (url->string uri))
199 (define (str->feed str)
200 (log-debug "Parsing feed string: ~v" str)
201 (match (string-split str)
206 (log-error "Invalid URI: ~v, exn: ~v" u e)
208 (feed nick (string->url u)))]
210 (log-error "Invalid feed string: ~v" str)
213 (define (filter-comments lines)
214 (filter-not (λ (line) (string-prefix? line "#")) lines))
216 (define (str->feeds str)
217 (filter-map str->feed (filter-comments (str->lines str))))
219 (define (file->feeds filename)
220 (str->feeds (file->string filename)))
225 [prog-version (info:#%info-lookup 'version)]
226 [prog-uri "https://github.com/xandkar/tt"]
227 [user-feed-file (expand-user-path "~/twtxt-me.txt")]
229 (if (file-exists? user-feed-file)
230 (let ([user (first (file->feeds user-feed-file))])
231 (format "+~a; @~a" (url->string (feed-uri user)) (feed-nick user)))
232 (format "+~a" prog-uri))])
233 (format "~a/~a (~a)" prog-name prog-version user)))
235 ; uri-download : net/url-structs:url -> Void
236 (define (uri-download u)
237 (define cache-file-path (url->cache-file-path u))
238 (log-debug "uri-download ~v into ~v" u cache-file-path)
239 (match* ((url-scheme u) (url-host u) (url-port u))
242 (define ssl? (string=? s "https"))
243 (define-values (status-line headers body-input)
244 ; TODO Timeout. Currently hangs on slow connections.
247 (url->string (struct-copy url u [scheme #f] [host #f]))
249 #:port (cond [p p] [ssl? 443] [else 80])
250 #:headers (list (format "User-Agent: ~a" user-agent))
252 (log-debug "headers: ~v" headers)
253 (log-debug "status-line: ~v" status-line)
255 (string->number (second (string-split (bytes->string/utf-8 status-line)))))
256 (log-debug "status: ~v" status)
257 ; TODO Handle redirects
259 (call-with-output-file cache-file-path
261 (copy-port body-input cache-output))
265 (log-error "Invalid URI: ~v" u)]))
267 (define (timeline-print out-format timeline)
268 (void (foldl (match-lambda**
269 [((and m (msg _ _ nick _ _)) (cons prev-nick i))
270 (let ([i (if (string=? prev-nick nick) i (+ 1 i))])
271 (msg-print out-format i m)
276 ; feed->msgs : Feed -> (Listof Msg)
277 (define (feed->msgs f)
278 (match-define (feed nick uri) f)
279 (log-info "Reading feed nick:~a uri:~v" nick uri)
280 (str->msgs nick uri (uri-read-cached uri)))
282 ; feed-download : Feed -> Void
283 (define (feed-download f)
284 (match-define (feed nick uri) f)
285 (log-info "Downloading feed nick:~a uri:~a" nick (url->string uri))
289 (log-error "Network error nick:~a uri:~v exn:~v" nick uri e)
293 (log-error "HTTP error nick:~a uri:~a status:~a" nick uri status)
297 ; timeline-download : Integer -> (Listof Feed) -> Void
298 (define (timeline-download num_workers feeds)
299 ; TODO No need for map - can just iter
300 (void (concurrent-filter-map num_workers feed-download feeds)))
302 ; TODO timeline contract : time-sorted list of messages
303 ; timeline-read : (U 'old->new 'new->old) -> (Listof Feeds) -> (Listof Msg)
304 (define (timeline-read order feeds)
305 (define cmp (match order
308 (sort (append* (filter-map feed->msgs feeds))
309 (λ (a b) (cmp (msg-ts_epoch a) (msg-ts_epoch b)))))
311 (define (start-logger level)
312 (let* ([logger (make-logger #f #f level #f)]
313 [log-receiver (make-log-receiver logger level)])
316 ([date-display-format 'iso-8601])
318 (define data (sync log-receiver))
319 (define level (vector-ref data 0))
320 (define msg (vector-ref data 1))
321 (define ts (date->string (current-date) #t))
322 (eprintf "~a [~a] ~a~n" ts level msg)
324 (current-logger logger)))
327 (let ([log-level 'info])
333 "Enable debug log level."
334 (set! log-level 'debug)]
337 "and <command> is one of"
338 "r, read i : Read the timeline."
339 "d, download : Download the timeline."
340 "u, upload : Upload your twtxt file (alias to execute ~/.tt/upload)."
342 #:args (command . args)
343 (start-logger log-level)
344 (current-command-line-arguments (list->vector args))
347 (let ([num_workers 15]) ; 15 was fastest out of the tried: 1, 5, 10, 20.
353 njobs "Number of concurrent jobs."
354 (set! num_workers (string->number njobs))]
356 (timeline-download num_workers (file->feeds filename))))]
362 (if (system (path->string (expand-user-path "~/.tt/upload")))
366 (let ([out-format 'multi-line]
373 "Reverse displayed timeline order."
374 (set! order 'new->old)]
377 "Short output format"
378 (set! out-format 'single-line)]
381 (set! out-format 'multi-line)]
383 (timeline-print out-format (timeline-read order (file->feeds filename)))))]