5 (require (prefix-in info: setup/getinfo))
19 uri ; net/url-structs:url
26 uri ; net/url-structs:url
29 (define (concurrent-filter-map num_workers f xs)
30 ; TODO preserve order of elements OR communicate that reorder is expected
31 ; TODO switch from mailboxes to channels
32 (define (make-worker id f)
33 (define parent (current-thread))
35 (define self (current-thread))
37 (thread-send parent (cons 'next self))
38 (match (thread-receive)
39 ['done (thread-send parent (cons 'exit id))]
40 [(cons 'unit x) (begin
42 (when y (thread-send parent (cons 'result y)))
45 (define (dispatch ws xs ys)
48 (match (thread-receive)
49 [(cons 'exit w) (dispatch (remove w ws =) xs ys)]
50 [(cons 'result y) (dispatch ws xs (cons y ys))]
51 [(cons 'next thd) (match xs
53 (thread-send thd 'done)
56 (thread-send thd (cons 'unit x))
57 (dispatch ws xs ys))])])))
58 (define workers (range num_workers))
59 (define threads (map (λ (id) (thread (make-worker id f))) workers))
60 (define results (dispatch workers xs '()))
61 (for-each thread-wait threads)
65 (let* ([f (λ (x) (if (even? x) x #f))]
67 [actual (sort (concurrent-filter-map 10 f xs) <)]
68 [expected (sort ( filter-map f xs) <)])
69 (check-equal? actual expected "concurrent-filter-map")))
72 (let* ([colors (vector 36 33)]
73 [n (vector-length colors)])
74 (λ (out-format color-i msg)
75 (let ([color (vector-ref colors (modulo color-i n))]
77 [uri (url->string (msg-uri msg))]
78 [text (msg-text msg)])
81 (printf "~a \033[1;37m<~a>\033[0m \033[0;~am~a\033[0m~n"
82 (parameterize ([date-display-format 'iso-8601])
83 (date->string (seconds->date [msg-ts_epoch msg]) #t))
86 (printf "~a~n\033[1;37m<~a ~a>\033[0m~n\033[0;~am~a\033[0m~n~n"
87 (parameterize ([date-display-format 'rfc2822])
88 (date->string (seconds->date [msg-ts_epoch msg]) #t))
89 nick uri color text)])))))
92 ; TODO Split parsing into 2 stages: 1) line->list; 2) rfc3339->epoch.
93 (let ([re (pregexp "^(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(:([0-9]{2}))?)(\\.[0-9]+)?([^\\s\t]*)[\\s\t]+(.*)$")])
99 "Failed to parse msg: ~v, from: ~v, at: ~v, because: ~v"
100 str nick (url->string uri) e)
102 (match (regexp-match re str)
103 [(list _wholething ts yyyy mm dd HH MM _:SS SS _f tz text)
105 ; TODO handle tz offset
106 ([ts_rfc3339 (string-append ts (if SS "" ":00") (if tz tz ""))]
107 [ts_epoch (find-seconds (if SS (string->number SS) 0)
112 (string->number yyyy))])
113 (msg ts_epoch ts_rfc3339 nick uri text))]
115 (log-debug "Non-msg line from nick:~a, line:~a" nick str)
119 (let* ([tzs (for*/list ([d '("-" "+")]
121 [m '("00" ":00" "57" ":57")])
122 (string-append d h m))]
123 [tzs (list* "" "Z" tzs)])
124 (for* ([n '("fake-nick")]
129 [sep (list "\t" " ")]
130 [txt '("foo bar baz" "'jaz poop bear giraffe / tea" "@*\"``")])
131 (let* ([ts (string-append "2020-11-18T22:22"
132 (if (non-empty-string? s) s ":00")
134 [m (str->msg n u (string-append ts sep txt))])
136 (check-equal? (msg-nick m) n)
137 (check-equal? (msg-uri m) u)
138 (check-equal? (msg-text m) txt)
139 (check-equal? (msg-ts_rfc3339 m) ts (format "Given: ~v" ts))
142 (let* ([ts "2020-11-18T22:22:09-0500"]
147 [actual (str->msg nick uri (string-append ts tab text))]
148 [expected (msg 1605756129 ts nick uri text)])
149 ; FIXME re-enable after handling tz offset
151 ; (msg-ts_epoch actual)
152 ; (msg-ts_epoch expected)
153 ; "str->msg ts_epoch")
155 (msg-ts_rfc3339 actual)
156 (msg-ts_rfc3339 expected)
157 "str->msg ts_rfc3339")
171 (define (str->lines str)
172 (string-split str (regexp "[\r\n]+")))
175 (check-equal? (str->lines "abc\ndef\n\nghi") '("abc" "def" "ghi")))
177 (define (str->msgs nick uri str)
178 (filter-map (λ (line) (str->msg nick uri line)) (str->lines str)))
180 (define (hash-sha1 str)
181 (define in (open-input-string str))
182 (define digest (sha1 in))
183 (close-input-port in)
186 (define (url->cache-file-path uri)
187 ; TODO Replace hashing with encoding
188 (expand-user-path (string-append "~/.tt/cache/" (hash-sha1 (url->string uri)))))
190 (define (uri-read-cached uri)
191 (define path (url->cache-file-path uri))
192 (if (file-exists? path)
195 (log-warning "Cache file not found for URI: ~a" (url->string uri))
198 (define (str->feed str)
199 (log-debug "Parsing feed string: ~v" str)
200 (match (string-split str)
205 (log-error "Invalid URI: ~v, exn: ~v" u e)
207 (feed nick (string->url u)))]
209 (log-error "Invalid feed string: ~v" str)
212 (define (filter-comments lines)
213 (filter-not (λ (line) (string-prefix? line "#")) lines))
215 (define (str->feeds str)
216 (filter-map str->feed (filter-comments (str->lines str))))
218 (define (file->feeds filename)
219 (str->feeds (file->string filename)))
224 [prog-version ((info:get-info (list prog-name)) 'version)]
225 [prog-uri "https://github.com/xandkar/tt"]
226 [user-feed-file (expand-user-path "~/twtxt-me.txt")]
228 (if (file-exists? user-feed-file)
229 (let ([user (first (file->feeds user-feed-file))])
230 (format "+~a; @~a" (url->string (feed-uri user)) (feed-nick user)))
231 (format "+~a" prog-uri))])
232 (format "~a/~a (~a)" prog-name prog-version user)))
234 ; uri-download : net/url-structs:url -> Void
235 (define (uri-download u)
236 (define cache-file-path (url->cache-file-path u))
237 (log-debug "uri-download ~v into ~v" u cache-file-path)
238 (match* ((url-scheme u) (url-host u) (url-port u))
241 (define ssl? (string=? s "https"))
242 (define-values (status-line headers body-input)
243 ; TODO Timeout. Currently hangs on slow connections.
246 (url->string (struct-copy url u [scheme #f] [host #f]))
248 #:port (cond [p p] [ssl? 443] [else 80])
249 #:headers (list (format "User-Agent: ~a" user-agent))
251 (log-debug "headers: ~v" headers)
252 (log-debug "status-line: ~v" status-line)
254 (string->number (second (string-split (bytes->string/utf-8 status-line)))))
255 (log-debug "status: ~v" status)
256 ; TODO Handle redirects
258 (call-with-output-file cache-file-path
260 (copy-port body-input cache-output))
264 (log-error "Invalid URI: ~v" u)]))
266 (define (timeline-print out-format timeline)
267 (void (foldl (match-lambda**
268 [((and m (msg _ _ nick _ _)) (cons prev-nick i))
269 (let ([i (if (string=? prev-nick nick) i (+ 1 i))])
270 (msg-print out-format i m)
275 ; feed->msgs : Feed -> (Listof Msg)
276 (define (feed->msgs f)
277 (match-define (feed nick uri) f)
278 (log-info "Reading feed nick:~a uri:~v" nick uri)
279 (str->msgs nick uri (uri-read-cached uri)))
281 ; feed-download : Feed -> Void
282 (define (feed-download f)
283 (match-define (feed nick uri) f)
284 (log-info "Downloading feed nick:~a uri:~a" nick (url->string uri))
288 (log-error "Network error nick:~a uri:~v exn:~v" nick uri e)
292 (log-error "HTTP error nick:~a uri:~a status:~a" nick uri status)
296 ; timeline-download : Integer -> (Listof Feed) -> Void
297 (define (timeline-download num_workers feeds)
298 ; TODO No need for map - can just iter
299 (void (concurrent-filter-map num_workers feed-download feeds)))
301 ; TODO timeline contract : time-sorted list of messages
302 ; timeline-read : (U 'old->new 'new->old) -> (Listof Feeds) -> (Listof Msg)
303 (define (timeline-read order feeds)
304 (define cmp (match order
307 (sort (append* (filter-map feed->msgs feeds))
308 (λ (a b) (cmp (msg-ts_epoch a) (msg-ts_epoch b)))))
310 (define (start-logger level)
311 (let* ([logger (make-logger #f #f level #f)]
312 [log-receiver (make-log-receiver logger level)])
315 ([date-display-format 'iso-8601])
317 (define data (sync log-receiver))
318 (define level (vector-ref data 0))
319 (define msg (vector-ref data 1))
320 (define ts (date->string (current-date) #t))
321 (eprintf "~a [~a] ~a~n" ts level msg)
323 (current-logger logger)))
326 (let ([log-level 'info])
332 "Enable debug log level."
333 (set! log-level 'debug)]
336 "and <command> is one of"
337 "r, read i : Read the timeline."
338 "d, download : Download the timeline."
339 "u, upload : Upload your twtxt file (alias to execute ~/.tt/upload)."
341 #:args (command . args)
342 (start-logger log-level)
343 (current-command-line-arguments (list->vector args))
346 (let ([num_workers 15]) ; 15 was fastest out of the tried: 1, 5, 10, 20.
352 njobs "Number of concurrent jobs."
353 (set! num_workers (string->number njobs))]
355 (timeline-download num_workers (file->feeds filename))))]
361 (if (system (path->string (expand-user-path "~/.tt/upload")))
365 (let ([out-format 'multi-line]
372 "Reverse displayed timeline order."
373 (set! order 'new->old)]
376 "Short output format"
377 (set! out-format 'single-line)]
380 (set! out-format 'multi-line)]
382 (timeline-print out-format (timeline-read order (file->feeds filename)))))]