X-Git-Url: https://git.xandkar.net/?a=blobdiff_plain;f=tt.rkt;h=d6218b8305d4c116014f189dc36e08f5a51ae822;hb=refs%2Ftags%2F0.16.0;hp=1cb90cf18bac9c964fdcb3a95dcc75b34c397778;hpb=9a346534e567f5496e2608b5946f1d21320bde94;p=tt.git

diff --git a/tt.rkt b/tt.rkt
index 1cb90cf..d6218b8 100644
--- a/tt.rkt
+++ b/tt.rkt
@@ -3,6 +3,7 @@
 (require openssl/sha1)
 (require racket/date)
 (require
+  net/head
   net/http-client
   net/uri-codec
   net/url-string
@@ -34,7 +35,8 @@
 
 (struct Peer
         ([nick : (Option String)]
-         [uri  : Url]))
+         [uri  : Url])
+        #:transparent)
 
 (: tt-home-dir Path-String)
 (define tt-home-dir (build-path (expand-user-path "~") ".tt"))
@@ -243,6 +245,8 @@
 (: cache-dir Path-String)
 (define cache-dir (build-path tt-home-dir "cache"))
 
+(define cache-object-dir (build-path cache-dir "objects"))
+
 (: url->cache-file-path-v1 (-> Url Path-String))
 (define (url->cache-file-path-v1 uri)
   (define (hash-sha1 str) : (-> String String)
@@ -250,13 +254,19 @@
     (define digest (sha1 in))
     (close-input-port in)
     digest)
-  (build-path cache-dir (hash-sha1 (url->string uri))))
+  (build-path cache-object-dir (hash-sha1 (url->string uri))))
 
 (: url->cache-file-path-v2 (-> Url Path-String))
 (define (url->cache-file-path-v2 uri)
-  (build-path cache-dir (uri-encode (url->string uri))))
+  (build-path cache-object-dir (uri-encode (url->string uri))))
+
+(define url->cache-object-path url->cache-file-path-v2)
+
+(define (url->cache-etag-path uri)
+  (build-path cache-dir "etags" (uri-encode (url->string uri))))
 
-(define url->cache-file-path url->cache-file-path-v2)
+(define (url->cache-lmod-path uri)
+  (build-path cache-dir "lmods" (uri-encode (url->string uri))))
 
 ; TODO Return Option
 (: uri-read-cached (-> Url String))
@@ -296,8 +306,12 @@
   (filter-map str->peer (filter-comments (str->lines str))))
 
 (: file->peers (-> Path-String (Listof Peer)))
-(define (file->peers filename)
-  (str->peers (file->string filename)))
+(define (file->peers file-path)
+  (if (file-exists? file-path)
+      (str->peers (file->string file-path))
+      (begin
+        (log-error "File does not exist: ~v" (path->string file-path))
+        '())))
 
 (: user-agent String)
 (define user-agent
@@ -314,10 +328,18 @@
            (format "+~a" prog-uri))])
     (format "~a/~a (~a)" prog-name prog-version user)))
 
+(: header-get (-> (Listof Bytes) Bytes (Option Bytes)))
+(define (header-get headers name)
+  (match (filter-map (curry extract-field name) headers)
+    [(list val) val]
+    [_           #f]))
+
 (: uri-download (-> Url Void))
 (define (uri-download u)
-  (define cache-file-path (url->cache-file-path u))
-  (log-debug "uri-download ~v into ~v" u cache-file-path)
+  (define cached-object-path (url->cache-object-path u))
+  (define cached-etag-path (url->cache-etag-path u))
+  (define cached-lmod-path (url->cache-lmod-path u))
+  (log-debug "uri-download ~v into ~v" u cached-object-path)
   (match* ((url-scheme u) (url-host u) (url-port u))
     [(s h p)
      #:when (and s h)
@@ -329,21 +351,38 @@
          (url->string (struct-copy url u [scheme #f] [host #f]))
          #:ssl? ssl?
          #:port (cond [p p] [ssl? 443] [else 80])
-         #:headers (list (format "User-Agent: ~a" user-agent))
-         ))
+         #:headers (list (format "User-Agent: ~a" user-agent))))
      (log-debug "headers: ~v" headers)
      (log-debug "status-line: ~v" status-line)
      (define status
        (string->number (second (string-split (bytes->string/utf-8 status-line)))))
      (log-debug "status: ~v" status)
      ; TODO Handle redirects
-     (if (= 200 status)
-         (begin
-           (make-parent-directory* cache-file-path)
-           (call-with-output-file cache-file-path
-                                  (curry copy-port body-input)
-                                  #:exists 'replace))
-         (raise status))]
+     (match status
+       [200
+         (let ([etag (header-get headers #"ETag")]
+               [lmod (header-get headers #"Last-Modified")])
+           (if (and etag
+                    (file-exists? cached-etag-path)
+                    (bytes=? etag (file->bytes cached-etag-path)))
+               (log-info "ETags match, skipping the rest of ~v" (url->string u))
+               (begin
+                 (log-info
+                   "Downloading the rest of ~v. ETag: ~a, Last-Modified: ~v"
+                   (url->string u) etag lmod)
+                 (make-parent-directory* cached-object-path)
+                 (make-parent-directory* cached-etag-path)
+                 (make-parent-directory* cached-lmod-path)
+                 (call-with-output-file cached-object-path
+                                        (curry copy-port body-input)
+                                        #:exists 'replace)
+                 (when etag
+                   (display-to-file etag cached-etag-path #:exists 'replace))
+                 (when lmod
+                   (display-to-file etag cached-lmod-path #:exists 'replace))))
+           (close-input-port body-input))]
+       [_
+         (raise status)])]
     [(_ _ _)
      (log-error "Invalid URI: ~v" u)]))
 
@@ -395,13 +434,29 @@
   (sort (append* (filter-map peer->msgs peers))
         (λ (a b) (cmp (Msg-ts-epoch a) (Msg-ts-epoch b)))))
 
+(: paths->peers (-> (Listof String) (Listof Peer)))
+(define (paths->peers paths)
+  (let* ([paths (match paths
+                  ['()
+                   (let ([peer-refs-file (build-path tt-home-dir "peers")])
+                     (log-debug
+                       "No peer ref file paths provided, defaulting to ~v"
+                       (path->string peer-refs-file))
+                     (list peer-refs-file))]
+                  [paths
+                    (log-debug "Peer ref file paths provided: ~v" paths)
+                    (map string->path paths)])]
+         [peers (append* (map file->peers paths))])
+    (log-info "Read-in ~a peers." (length peers))
+    peers))
+
 (: log-writer-stop (-> Thread Void))
 (define (log-writer-stop log-writer)
   (log-message (current-logger) 'fatal 'stop "Exiting." #f)
   (thread-wait log-writer))
 
-(: logger-start (-> Log-Level Thread))
-(define (logger-start level)
+(: log-writer-start (-> Log-Level Thread))
+(define (log-writer-start level)
   (let* ([logger
            (make-logger #f #f level #f)]
          [log-receiver
@@ -437,7 +492,7 @@
       "u, upload   : Upload your twtxt file (alias to execute ~/.tt/upload)."
       ""
       #:args (command . args)
-      (define log-writer (logger-start log-level))
+      (define log-writer (log-writer-start log-level))
       (current-command-line-arguments (list->vector args))
       (match command
         [(or "d" "download")
@@ -451,11 +506,10 @@
              [("-j" "--jobs")
               njobs "Number of concurrent jobs."
               (set! num-workers (string->number njobs))]
-             #:args (filename)
+             #:args file-paths
              (define-values (_res _cpu real-ms _gc)
-               (time-apply timeline-download (list num-workers (file->peers filename))))
-             (log-info "Timeline downloaded in ~a seconds." (/ real-ms 1000.0))
-             (log-writer-stop log-writer)))]
+               (time-apply timeline-download (list num-workers (paths->peers file-paths))))
+             (log-info "Timeline downloaded in ~a seconds." (/ real-ms 1000.0))))]
         [(or "u" "upload")
          (command-line
            #:program
@@ -481,6 +535,6 @@
              [("-l" "--long")
               "Long output format"
               (set! out-format 'multi-line)]
-             #:args (filename)
-             (timeline-print out-format (timeline-read order (file->peers filename)))))]
-        ))))
+             #:args file-paths
+             (timeline-print out-format (timeline-read order (paths->peers file-paths)))))])
+      (log-writer-stop log-writer))))