From: Siraaj Khandkar Date: Wed, 13 Mar 2019 10:27:12 +0000 (-0400) Subject: Add web-cloning scripts X-Git-Url: https://git.xandkar.net/?a=commitdiff_plain;ds=inline;h=06ec33cbfa90c35e1c507bf54bba8a3b01b58f90;p=khome.git Add web-cloning scripts --- diff --git a/bin/wget-clone-web b/bin/wget-clone-web new file mode 100755 index 0000000..1f19c0e --- /dev/null +++ b/bin/wget-clone-web @@ -0,0 +1,48 @@ +#! /bin/sh + +uri_and_extra_options=$@; + + +# Continue partially downloaded +OPT_CONTINUE='-c' + +OPT_WAIT='--wait 0.75' + +# Randomly select wait (above) in range from wait*0.5 to wait*1.5 +OPT_WAIT_RANDON='--random-wait' + +# --page-requisites "download all the files that are necessary to properly +# display a given HTML page." +OPT_PAGE_REQUISITES='-p' + +# --user-agent="" +#OPT_USER_AGENT="-U='Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0'" +OPT_USER_AGENT="-U='ELinks (0.12pre6; NetBSD; 800x600)'" + +# --convert-links +# After the download is complete, convert the links in the document to make +# them suitable for local viewing. This affects not only the visible +# hyperlinks, but any part of the document that links to external content, such +# as embedded images, links to style sheets, hyperlinks to non-HTML content, +# etc. +OPT_LINKS_LOCAL='-k' + +# --adjust-extension +# Ensure file extension matches file's MIME type +OPT_EXT_ADJUST='-E' + +# Disrespect robots.txt +OPT_ROBOTS_OFF='-e robots=off' + + +wget \ + -a wget.log \ + $OPT_CONTINUE \ + $OPT_WAIT \ + $OPT_WAIT_RANDON \ + $OPT_PAGE_REQUISITES \ + $OPT_LINKS_LOCAL \ + $OPT_EXT_ADJUST \ + "$OPT_USER_AGENT" \ + $OPT_ROBOTS_OFF \ + $uri_and_extra_options diff --git a/bin/wget-clone-web-page b/bin/wget-clone-web-page new file mode 100755 index 0000000..4de0d72 --- /dev/null +++ b/bin/wget-clone-web-page @@ -0,0 +1,3 @@ +#! /bin/sh + +wget-clone-web $@ diff --git a/bin/wget-clone-web-site b/bin/wget-clone-web-site new file mode 100755 index 0000000..0ae4419 --- /dev/null +++ b/bin/wget-clone-web-site @@ -0,0 +1,6 @@ +#! /bin/sh + +# Recursively download the site +OPT_RECURSIVE='-r' + +wget-clone-web $OPT_RECURSIVE $@