Next: , Previous: , Up: Integration with existing software  


Downloading service

Previous sections tell about manual downloading and sending results to remote node. But one wish to remotely initiate downloading. That can be easily solved with exec handles.

exec: {
  warcer: ["/bin/sh", "/path/to/warcer.sh"]
  wgeter: ["/bin/sh", "/path/to/wgeter.sh"]
  aria2c: [
    "/usr/local/bin/aria2c",
    "--on-download-complete", "aria2-downloaded.sh",
    "--on-bt-download-complete", "aria2-downloaded.sh"
  ]
}

warcer.sh contents:

#!/bin/sh -ex

name="$1"-$(date '+%Y%M%d%H%m%S')
read cmdline

tmp=$(mktemp -d)
cd $tmp
wget \
    --page-requisites \
    --convert-links \
    --adjust-extension \
    --restrict-file-names=ascii \
    --span-hosts \
    --random-wait \
    --execute robots=off \
    --reject '*.woff*,*.ttf,*.eot,*.js' \
    --tries 10 \
    --warc-file "$name" \
    --no-warc-compression \
    --no-warc-keep-log \
    $cmdline || :
zstd --rm "$name".warc
nncp-file -nice $NNCP_NICE "$name".warc.zst $NNCP_SENDER:
rm -r $tmp

wgeter.sh contents:

#!/bin/sh -ex

name=$1-$(date '+%Y%M%d%H%m%S')
read cmdline

tmp=$(mktemp)
wget --output-document=$tmp $cmdline
zstd --rm $tmp
nncp-file -nice $NNCP_NICE $tmp.zst $NNCP_SENDER:$name.zst
rm $tmp.zst

Now you can queue that node to send you some website’s page, file or BitTorrents:

$ echo http://www.nncpgo.org/Postfix.html |
    nncp-exec remote.node warcer postfix-whole-page
$ echo http://www.nncpgo.org/Postfix.html |
    nncp-exec remote.node wgeter postfix-html-page
$ echo \
    http://www.nncpgo.org/download/nncp-0.11.tar.xz
    http://www.nncpgo.org/download/nncp-0.11.tar.xz.sig |
    nncp-exec remote.node aria2c