From e09dff7f9b5875b81cf50b33dcf97635518f7c28 Mon Sep 17 00:00:00 2001
From: Siraaj Khandkar <siraaj@khandkar.net>
Date: Wed, 14 Nov 2018 15:17:55 -0500
Subject: [PATCH] Support outputting to files in a directory

---
 dups.ml | 115 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 77 insertions(+), 38 deletions(-)

diff --git a/dups.ml b/dups.ml
index ba1fb0c..b6782b6 100644
--- a/dups.ml
+++ b/dups.ml
@@ -3,6 +3,7 @@ open Printf
 module Array = ArrayLabels
 module List  = ListLabels
 module StrSet= Set.Make(String)
+module Unix  = UnixLabels
 
 module Stream : sig
   type 'a t
@@ -80,60 +81,98 @@ type input =
   | Root_paths of string list
   | Paths_on_stdin
 
-let main input =
-  let paths =
-    match input with
-    | Paths_on_stdin ->
-        In_channel.lines stdin
-    | Root_paths paths ->
-        let paths = StrSet.elements (StrSet.of_list paths) in
-        Stream.concat (List.map paths ~f:Directory_tree.find_files)
-  in
+type output =
+  | Stdout
+  | Directory of string
+
+let make_input_stream = function
+  | Paths_on_stdin ->
+      In_channel.lines stdin
+  | Root_paths paths ->
+      let paths = StrSet.elements (StrSet.of_list paths) in
+      Stream.concat (List.map paths ~f:Directory_tree.find_files)
+
+let make_output_fun = function
+  | Stdout ->
+      fun digest n_paths paths ->
+        printf "%s %d\n%!" (Digest.to_hex digest) n_paths;
+        List.iter (StrSet.elements paths) ~f:(printf "    %S\n%!")
+  | Directory dir ->
+      fun digest _ paths ->
+        let digest = Digest.to_hex digest in
+        let dir = Filename.concat dir (String.sub digest 0 2) in
+        Unix.mkdir dir ~perm:0o700;
+        let oc = open_out (Filename.concat dir digest) in
+        List.iter (StrSet.elements paths) ~f:(fun path ->
+          output_string oc (sprintf "%S\n%!" path)
+        );
+        close_out oc
+
+let main input output =
+  let output = make_output_fun  output in
+  let input  = make_input_stream input in
   let paths_by_digest = Hashtbl.create 1_000_000 in
   let path_count = ref 0 in
   let t0 = Sys.time () in
-  Stream.iter paths ~f:(fun path ->
+  Stream.iter input ~f:(fun path ->
     incr path_count;
     try
       let digest = Digest.file path in
-      let paths =
+      let count, paths =
         match Hashtbl.find_opt paths_by_digest digest with
         | None ->
-            StrSet.empty
-        | Some paths ->
-            paths
+            (0, StrSet.empty)
+        | Some (n, paths) ->
+            (n, paths)
       in
-      Hashtbl.replace paths_by_digest digest (StrSet.add path paths)
+      Hashtbl.replace paths_by_digest digest (count + 1, StrSet.add path paths)
     with Sys_error e ->
       eprintf "WARNING: Failed to process %S: %S\n%!" path e
   );
-  Hashtbl.iter
-    (fun digest paths ->
-      let n_paths = StrSet.cardinal paths in
-      if n_paths > 1 then begin
-        printf "%s %d\n%!" (Digest.to_hex digest) n_paths;
-        List.iter (StrSet.elements paths) ~f:(printf "    %S\n%!")
-      end
-    )
-    paths_by_digest;
+  Hashtbl.iter (fun d (n, ps) -> if n > 1 then output d n ps) paths_by_digest;
   let t1 = Sys.time () in
   eprintf "Processed %d files in %f seconds.\n%!" !path_count (t1 -. t0)
 
 let () =
-  let input = ref Paths_on_stdin in
-  Arg.parse
-    []
-    (function
-    | path when Sys.file_exists path ->
-        (match !input with
-        | Paths_on_stdin ->
-            input := Root_paths [path]
-        | Root_paths paths ->
-            input := Root_paths (path :: paths)
+  let input  = ref Paths_on_stdin in
+  let output = ref Stdout in
+  let assert_file_exists path =
+    if Sys.file_exists path then
+      ()
+    else begin
+      eprintf "File does not exist: %S\n%!" path;
+      exit 1
+    end
+  in
+  let assert_file_is_dir path =
+    if Sys.is_directory path then
+      ()
+    else begin
+      eprintf "File is not a directory: %S\n%!" path;
+      exit 1
+    end
+  in
+  let spec =
+    [ ( "-out"
+      , Arg.String (fun path ->
+          assert_file_exists path;
+          assert_file_is_dir path;
+          output := Directory path
         )
-    | path ->
-        eprintf "File does not exist: %S\n%!" path;
-        exit 1
+      , " Output to this directory instead of stdout."
+      )
+    ]
+  in
+  Arg.parse
+    (Arg.align spec)
+    (fun path ->
+      assert_file_exists path;
+      assert_file_is_dir path;
+      match !input with
+      | Paths_on_stdin ->
+          input := Root_paths [path]
+      | Root_paths paths ->
+          input := Root_paths (path :: paths)
     )
     "";
-  main !input
+  main !input !output
-- 
2.20.1