5 (require (prefix-in info: setup/getinfo))
21 uri ; net/url-structs:url
28 uri ; net/url-structs:url
31 (define (concurrent-filter-map num_workers f xs)
32 ; TODO preserve order of elements OR communicate that reorder is expected
33 ; TODO switch from mailboxes to channels
34 (define (make-worker id f)
35 (define parent (current-thread))
37 (define self (current-thread))
39 (thread-send parent (cons 'next self))
40 (match (thread-receive)
41 ['done (thread-send parent (cons 'exit id))]
42 [(cons 'unit x) (begin
44 (when y (thread-send parent (cons 'result y)))
47 (define (dispatch ws xs ys)
50 (match (thread-receive)
51 [(cons 'exit w) (dispatch (remove w ws =) xs ys)]
52 [(cons 'result y) (dispatch ws xs (cons y ys))]
53 [(cons 'next thd) (match xs
55 (thread-send thd 'done)
58 (thread-send thd (cons 'unit x))
59 (dispatch ws xs ys))])])))
60 (define workers (range num_workers))
61 (define threads (map (λ (id) (thread (make-worker id f))) workers))
62 (define results (dispatch workers xs '()))
63 (for-each thread-wait threads)
67 (let* ([f (λ (x) (if (even? x) x #f))]
69 [actual (sort (concurrent-filter-map 10 f xs) <)]
70 [expected (sort ( filter-map f xs) <)])
71 (check-equal? actual expected "concurrent-filter-map")))
74 (let* ([colors (vector 36 33)]
75 [n (vector-length colors)])
76 (λ (out-format color-i msg)
77 (let ([color (vector-ref colors (modulo color-i n))]
79 [uri (url->string (msg-uri msg))]
80 [text (msg-text msg)])
83 (printf "~a \033[1;37m<~a>\033[0m \033[0;~am~a\033[0m~n"
84 (parameterize ([date-display-format 'iso-8601])
85 (date->string (seconds->date [msg-ts_epoch msg]) #t))
88 (printf "~a~n\033[1;37m<~a ~a>\033[0m~n\033[0;~am~a\033[0m~n~n"
89 (parameterize ([date-display-format 'rfc2822])
90 (date->string (seconds->date [msg-ts_epoch msg]) #t))
91 nick uri color text)])))))
93 ; TODO Implement rfc3339->epoch and remove dependency on rfc3339-old
96 (let ([re (pregexp "^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?)(\\.[0-9]+)?([^\\s\t]*)[\\s\t]+(.*)$")])
102 "Failed to parse msg: ~v, from: ~v, at: ~v, because: ~v"
103 str nick (url->string uri) e)
105 (match (regexp-match re str)
106 [(list _wholething ts s _fractional tz text)
108 ([ts_rfc3339 (string-append ts (if s "" ":00") (if tz tz ""))]
109 [t (string->rfc3339-record ts_rfc3339)]
110 [s (rfc3339-record:second t)]
111 ; TODO handle tz offset
112 [ts_epoch (find-seconds [if s s 0]
113 [rfc3339-record:minute t]
114 [rfc3339-record:hour t]
115 [rfc3339-record:mday t]
116 [rfc3339-record:month t]
117 [rfc3339-record:year t])])
118 (msg ts_epoch ts_rfc3339 nick uri text))]
120 (log-debug "Non-msg line from nick:~a, line:~a" nick str)
124 (let* ([tzs (for*/list ([d '("-" "+")]
126 [m '("00" ":00" "57" ":57")])
127 (string-append d h m))]
128 [tzs (list* "" "Z" tzs)])
129 (for* ([n '("fake-nick")]
134 [sep (list "\t" " ")]
135 [txt '("foo bar baz" "'jaz poop bear giraffe / tea" "@*\"``")])
136 (let* ([ts (string-append "2020-11-18T22:22"
137 (if (non-empty-string? s) s ":00")
139 [m (str->msg n u (string-append ts sep txt))])
141 (check-equal? (msg-nick m) n)
142 (check-equal? (msg-uri m) u)
143 (check-equal? (msg-text m) txt)
144 (check-equal? (msg-ts_rfc3339 m) ts (format "Given: ~v" ts))
147 (let* ([ts "2020-11-18T22:22:09-0500"]
152 [actual (str->msg nick uri (string-append ts tab text))]
153 [expected (msg 1605756129 ts nick uri text)])
154 ; FIXME re-enable after handling tz offset
156 ; (msg-ts_epoch actual)
157 ; (msg-ts_epoch expected)
158 ; "str->msg ts_epoch")
160 (msg-ts_rfc3339 actual)
161 (msg-ts_rfc3339 expected)
162 "str->msg ts_rfc3339")
176 (define (str->lines str)
177 (string-split str (regexp "[\r\n]+")))
180 (check-equal? (str->lines "abc\ndef\n\nghi") '("abc" "def" "ghi")))
182 (define (str->msgs nick uri str)
183 (filter-map (λ (line) (str->msg nick uri line)) (str->lines str)))
185 (define (hash-sha1 str)
186 (define in (open-input-string str))
187 (define digest (sha1 in))
188 (close-input-port in)
191 (define (url->cache-file-path uri)
192 ; TODO Replace hashing with encoding
193 (expand-user-path (string-append "~/.tt/cache/" (hash-sha1 (url->string uri)))))
195 (define (uri-read-cached uri)
196 (define path (url->cache-file-path uri))
197 (if (file-exists? path)
200 (log-warning "Cache file not found for URI: ~a" (url->string uri))
203 (define (str->feed str)
204 (log-debug "Parsing feed string: ~v" str)
205 (match (string-split str)
210 (log-error "Invalid URI: ~v, exn: ~v" u e)
212 (feed nick (string->url u)))]
214 (log-error "Invalid feed string: ~v" str)
217 (define (filter-comments lines)
218 (filter-not (λ (line) (string-prefix? line "#")) lines))
220 (define (str->feeds str)
221 (filter-map str->feed (filter-comments (str->lines str))))
223 (define (file->feeds filename)
224 (str->feeds (file->string filename)))
229 [prog-version ((info:get-info (list prog-name)) 'version)]
230 [prog-uri "https://github.com/xandkar/tt"]
231 [user-feed-file (expand-user-path "~/twtxt-me.txt")]
233 (if (file-exists? user-feed-file)
234 (let ([user (first (file->feeds user-feed-file))])
235 (format "+~a; @~a" (url->string (feed-uri user)) (feed-nick user)))
236 (format "+~a" prog-uri))])
237 (format "~a/~a (~a)" prog-name prog-version user)))
239 ; uri-download : net/url-structs:url -> Void
240 (define (uri-download u)
241 (define cache-file-path (url->cache-file-path u))
242 (log-debug "uri-download ~v into ~v" u cache-file-path)
243 (match* ((url-scheme u) (url-host u) (url-port u))
246 (define ssl? (string=? s "https"))
247 (define-values (status-line headers body-input)
248 ; TODO Timeout. Currently hangs on slow connections.
251 (url->string (struct-copy url u [scheme #f] [host #f]))
253 #:port (cond [p p] [ssl? 443] [else 80])
254 #:headers (list (format "User-Agent: ~a" user-agent))
256 (log-debug "headers: ~v" headers)
257 (log-debug "status-line: ~v" status-line)
259 (string->number (second (string-split (bytes->string/utf-8 status-line)))))
260 (log-debug "status: ~v" status)
261 ; TODO Handle redirects
263 (call-with-output-file cache-file-path
265 (copy-port body-input cache-output))
269 (log-error "Invalid URI: ~v" u)]))
271 (define (timeline-print out-format timeline)
272 (void (foldl (match-lambda**
273 [((and m (msg _ _ nick _ _)) (cons prev-nick i))
274 (let ([i (if (string=? prev-nick nick) i (+ 1 i))])
275 (msg-print out-format i m)
280 ; feed->msgs : Feed -> (Listof Msg)
281 (define (feed->msgs f)
282 (match-define (feed nick uri) f)
283 (log-info "Reading feed nick:~a uri:~v" nick uri)
284 (str->msgs nick uri (uri-read-cached uri)))
286 ; feed-download : Feed -> Void
287 (define (feed-download f)
288 (match-define (feed nick uri) f)
289 (log-info "Downloading feed nick:~a uri:~a" nick (url->string uri))
293 (log-error "Network error nick:~a uri:~v exn:~v" nick uri e)
297 (log-error "HTTP error nick:~a uri:~a status:~a" nick uri status)
301 ; timeline-download : Integer -> (Listof Feed) -> Void
302 (define (timeline-download num_workers feeds)
303 ; TODO No need for map - can just iter
304 (void (concurrent-filter-map num_workers feed-download feeds)))
306 ; TODO timeline contract : time-sorted list of messages
307 ; timeline-read : (U 'old->new 'new->old) -> (Listof Feeds) -> (Listof Msg)
308 (define (timeline-read order feeds)
309 (define cmp (match order
312 (sort (append* (filter-map feed->msgs feeds))
313 (λ (a b) (cmp (msg-ts_epoch a) (msg-ts_epoch b)))))
315 (define (start-logger level)
316 (let* ([logger (make-logger #f #f level #f)]
317 [log-receiver (make-log-receiver logger level)])
320 ([date-display-format 'iso-8601])
322 (define data (sync log-receiver))
323 (define level (vector-ref data 0))
324 (define msg (vector-ref data 1))
325 (define ts (date->string (current-date) #t))
326 (eprintf "~a [~a] ~a~n" ts level msg)
328 (current-logger logger)))
331 (let ([log-level 'info])
337 "Enable debug log level."
338 (set! log-level 'debug)]
341 "and <command> is one of"
342 "r, read i : Read the timeline."
343 "d, download : Download the timeline."
344 "u, upload : Upload your twtxt file (alias to execute ~/.tt/upload)."
346 #:args (command . args)
347 (start-logger log-level)
348 (current-command-line-arguments (list->vector args))
351 (let ([num_workers 15]) ; 15 was fastest out of the tried: 1, 5, 10, 20.
357 njobs "Number of concurrent jobs."
358 (set! num_workers (string->number njobs))]
360 (timeline-download num_workers (file->feeds filename))))]
366 (if (system (path->string (expand-user-path "~/.tt/upload")))
370 (let ([out-format 'multi-line]
377 "Reverse displayed timeline order."
378 (set! order 'new->old)]
381 "Short output format"
382 (set! out-format 'single-line)]
385 (set! out-format 'multi-line)]
387 (timeline-print out-format (timeline-read order (file->feeds filename)))))]