3 module Array = ArrayLabels
4 module List = ListLabels
5 module StrSet= Set.Make(String)
10 val create : (unit -> 'a option) -> 'a t
12 val iter : 'a t -> f:('a -> unit) -> unit
14 val concat : ('a t) list -> 'a t
22 [S.from (fun _ -> f ())]
25 List.iter t ~f:(S.iter f)
31 module In_channel : sig
32 val lines : in_channel -> string Stream.t
35 Stream.create (fun () ->
36 match input_line ic with
37 | exception End_of_file ->
44 module Directory_tree : sig
45 val find_files : string -> string Stream.t
48 let dirs = Queue.create () in
49 let files = Queue.create () in
51 Array.iter (Sys.readdir parent) ~f:(fun child ->
52 let path = Filename.concat parent child in
53 let {Unix.st_kind = file_kind; _} = Unix.lstat path in
69 match Queue.is_empty files, Queue.is_empty dirs with
70 | false, _ -> Some (Queue.take files)
73 explore (Queue.take dirs);
80 | Root_paths of string list
87 In_channel.lines stdin
89 let paths = StrSet.elements (StrSet.of_list paths) in
90 Stream.concat (List.map paths ~f:Directory_tree.find_files)
92 let paths_by_digest = Hashtbl.create 1_000_000 in
93 let path_count = ref 0 in
94 let t0 = Sys.time () in
95 Stream.iter paths ~f:(fun path ->
98 let digest = Digest.file path in
100 match Hashtbl.find_opt paths_by_digest digest with
106 Hashtbl.replace paths_by_digest digest (StrSet.add path paths)
108 eprintf "WARNING: Failed to process %S: %S\n%!" path e
112 let n_paths = StrSet.cardinal paths in
113 if n_paths > 1 then begin
114 printf "%s %d\n%!" (Digest.to_hex digest) n_paths;
115 List.iter (StrSet.elements paths) ~f:(printf " %S\n%!")
119 let t1 = Sys.time () in
120 eprintf "Processed %d files in %f seconds.\n%!" !path_count (t1 -. t0)
123 let input = ref Paths_on_stdin in
127 | path when Sys.file_exists path ->
130 input := Root_paths [path]
131 | Root_paths paths ->
132 input := Root_paths (path :: paths)
135 eprintf "File does not exist: %S\n%!" path;