initial commit

node_modules/nugget/.npmignore

@@ -0,0 +1,2 @@
+.DS_Store
+node_modules

node_modules/nugget/.travis.yml

@@ -0,0 +1,3 @@
+language: node_js
+node_js:
+  - '0.10'

node_modules/nugget/bin.js

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+
+var nugget = require('./')
+var args = require('minimist')(process.argv.slice(2))
+
+var urls = args._
+if (urls.length === 0) {
+  console.log(
+    "Usage: nugget <urls> [options]\n" +
+    "  -o    output filename\n" +
+    "  -d    output parent directory\n" +
+    "  -c    resume aborted download\n" +
+    "  -f    ignore response codes > 299\n" +
+    "  -s    concurrent socket limit (default infinity)"
+  )
+  process.exit(1)
+}
+
+var opts = {
+  target:    args.o || args.O || args.out,
+  dir:       args.d || args.dir,
+  resume:    args.c || args.continue,
+  force:     args.f || args.force,
+  sockets:   args.s || args.sockets,
+  verbose:   args.verbose === undefined ? process.stdin.isTTY : args.verbose,
+  frequency: args.frequency ? +args.frequency : null,
+  proxy:     args.proxy ? args.proxy : null,
+  strictSSL: args['strict-ssl']
+}
+
+nugget(urls, opts, function(err) {
+  if (err) {
+    console.error('Error:', err)
+    process.exit(1)
+  }
+  process.exit(0)
+})

node_modules/nugget/collaborators.md

@@ -0,0 +1,9 @@
+## Collaborators
+
+nugget is only possible due to the excellent work of the following collaborators:
+
+<table><tbody><tr><th align="left">maxogden</th><td><a href="https://github.com/maxogden">GitHub/maxogden</a></td></tr>
+<tr><th align="left">grncdr</th><td><a href="https://github.com/grncdr">GitHub/grncdr</a></td></tr>
+<tr><th align="left">mafintosh</th><td><a href="https://github.com/mafintosh">GitHub/mafintosh</a></td></tr>
+<tr><th align="left">jlord</th><td><a href="https://github.com/jlord">GitHub/jlord</a></td></tr>
+</tbody></table>

node_modules/nugget/index.js

@@ -0,0 +1,204 @@
+var request = require('request')
+var fs = require('fs')
+var path = require('path')
+var log = require('single-line-log').stdout
+var progress = require('progress-stream')
+var prettyBytes = require('pretty-bytes')
+var throttle = require('throttleit')
+var EventEmitter = require('events').EventEmitter
+var debug = require('debug')('nugget')
+
+function noop () {}
+
+module.exports = function(urls, opts, cb) {
+  if (!Array.isArray(urls)) urls = [urls]
+  if (urls.length === 1) opts.singleTarget = true
+
+  var defaultProps = {};
+
+  if (opts.sockets) {
+    var sockets = +opts.sockets
+    defaultProps.pool = {maxSockets: sockets}
+  }
+
+  if (opts.proxy) {
+    defaultProps.proxy = opts.proxy
+  }
+
+  if (opts.strictSSL !== null) {
+    defaultProps.strictSSL = opts.strictSSL
+  }
+
+  if (Object.keys(defaultProps).length > 0) {
+    request = request.defaults(defaultProps);
+  }
+
+  var downloads = []
+  var errors = []
+  var pending = 0
+  var truncated = urls.length * 2 >= (process.stdout.rows - 15)
+
+  urls.forEach(function (url) {
+    debug('start dl', url)
+    pending++
+    var dl = startDownload(url, opts, function done (err) {
+      debug('done dl', url, pending)
+      if (err) {
+        debug('error dl', url, err)
+        errors.push(err)
+        dl.error = err.message
+      }
+      if (truncated) {
+        var i = downloads.indexOf(dl)
+        downloads.splice(i, 1)
+        downloads.push(dl)
+      }
+      if (--pending === 0) {
+        render()
+        cb(errors.length ? errors : undefined)
+      }
+    })
+
+    downloads.push(dl)
+
+    dl.on('start', function (progressStream) {
+      throttledRender()
+    })
+
+    dl.on('progress', function(data) {
+      debug('progress', url, data.percentage)
+
+      dl.speed = data.speed
+      if (dl.percentage === 100) render()
+      else throttledRender()
+    })
+  })
+
+  var _log = opts.verbose ? log : noop
+  render()
+  var throttledRender = throttle(render, opts.frequency || 250)
+
+  if (opts.singleTarget) return downloads[0]
+  else return downloads
+
+  function render () {
+    var height = process.stdout.rows
+    var rendered = 0
+    var output = ""
+    var totalSpeed = 0
+    downloads.forEach(function (dl) {
+      if (2 * rendered >= height - 15) return
+      rendered++
+      if (dl.error) {
+        output += 'Downloading '+path.basename(dl.target)+'\n'
+        output += 'Error: ' + dl.error + '\n'
+        return
+      }
+      var pct = dl.percentage
+      var speed = dl.speed
+      var total = dl.fileSize
+      totalSpeed += speed
+      var bar = Array(Math.floor(45 * pct / 100)).join('=')+'>'
+      while (bar.length < 45) bar += ' '
+      output += 'Downloading '+path.basename(dl.target)+'\n'+
+      '['+bar+'] '+pct.toFixed(1)+'%'
+      if (total) output += ' of ' + prettyBytes(total)
+      output += ' (' + prettyBytes(speed) + '/s)\n'
+    })
+    if (rendered < downloads.length) output += '\n... and ' + (downloads.length - rendered) + ' more\n'
+    if (downloads.length > 1) output += '\nCombined Speed: ' + prettyBytes(totalSpeed) + '/s\n'
+    _log(output)
+  }
+
+  function startDownload (url, opts, cb) {
+    var targetName = path.basename(url).split('?')[0]
+    if (opts.singleTarget && opts.target) targetName = opts.target
+    var target = path.resolve(opts.dir || process.cwd(), targetName)
+    if (opts.resume) {
+      resume(url, opts, cb)
+    } else {
+      download(url, opts, cb)
+    }
+
+    var progressEmitter = new EventEmitter()
+    progressEmitter.target = target
+    progressEmitter.speed = 0
+    progressEmitter.percentage = 0
+
+    return progressEmitter
+
+    function resume(url, opts, cb) {
+      fs.stat(target, function (err, stats) {
+        if (err && err.code === 'ENOENT') {
+          return download(url, opts, cb)
+        }
+        if (err) {
+          return cb(err)
+        }
+        var offset = stats.size
+        var req = request.get(url)
+
+        req.on('error', cb)
+        req.on('response', function(resp) {
+          resp.destroy()
+
+          var length = parseInt(resp.headers['content-length'], 10)
+
+          // file is already downloaded.
+          if (length === offset) return cb()
+
+          if (!isNaN(length) && length > offset && /bytes/.test(resp.headers['accept-ranges'])) {
+            opts.range = [offset, length]
+          }
+
+          download(url, opts, cb)
+        })
+      })
+    }
+
+    function download(url, opts, cb) {
+      var headers = opts.headers || {}
+      if (opts.range) {
+        headers.Range = 'bytes=' + opts.range[0] + '-' + opts.range[1]
+      }
+      var read = request(url, { headers: headers })
+      var speed = "0 Kb"
+
+      read.on('error', cb)
+      read.on('response', function(resp) {
+        debug('response', url, resp.statusCode)
+        if (resp.statusCode > 299 && !opts.force) return cb(new Error('GET ' + url + ' returned ' + resp.statusCode))
+        var write = fs.createWriteStream(target, {flags: opts.resume ? 'a' : 'w'})
+        write.on('error', cb)
+        write.on('finish', cb)
+
+        var fullLen
+        var contentLen = Number(resp.headers['content-length'])
+        var range = resp.headers['content-range']
+        if (range) {
+          fullLen = Number(range.split('/')[1])
+        } else {
+          fullLen = contentLen
+        }
+
+        progressEmitter.fileSize = fullLen
+        if (range) {
+          var downloaded = fullLen - contentLen
+        }
+        var progressStream = progress({ length: fullLen, transferred: downloaded }, onprogress)
+        progressEmitter.emit('start', progressStream)
+
+        resp
+          .pipe(progressStream)
+          .pipe(write)
+      })
+
+      function onprogress (p) {
+        var pct = p.percentage
+        progressEmitter.progress = p
+        progressEmitter.percentage = pct
+        progressEmitter.emit('progress', p)
+      }
+    }
+  }
+}

node_modules/nugget/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "nugget",
+  "version": "1.6.2",
+  "description": "minimalist wget clone written in node. HTTP GETs a file and saves it to the current working directory",
+  "main": "index.js",
+  "bin": {
+    "nugget": "bin.js"
+  },
+  "scripts": {
+    "test": "tape test/*.js"
+  },
+  "author": "max ogden",
+  "license": "BSD",
+  "dependencies": {
+    "debug": "^2.1.3",
+    "minimist": "^1.1.0",
+    "pretty-bytes": "^1.0.2",
+    "progress-stream": "^1.1.0",
+    "request": "^2.45.0",
+    "single-line-log": "^0.4.1",
+    "throttleit": "0.0.2"
+  },
+  "devDependencies": {
+    "tape": "^3.0.1"
+  },
+  "directories": {
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/maxogden/nugget.git"
+  },
+  "bugs": {
+    "url": "https://github.com/maxogden/nugget/issues"
+  },
+  "homepage": "https://github.com/maxogden/nugget"
+}

node_modules/nugget/readme.md

@@ -0,0 +1,45 @@
+# nugget
+
+Minimalist command line downloader written in node, inspired by wget. HTTP GETs a file and streams it into a file in the current working directory. Specializes at downloading many files in parallel.
+
+[![NPM](https://nodei.co/npm/nugget.png?global=true)](https://nodei.co/npm/nugget/)
+![dat](http://img.shields.io/badge/Development%20sponsored%20by-dat-green.svg?style=flat)
+[![Travis](http://img.shields.io/travis/maxogden/nugget.svg?style=flat)](https://travis-ci.org/maxogden/nugget)
+
+## installation
+
+```
+npm install nugget -g
+```
+
+## usage
+
+```
+nugget http://foo.com/bar.jpg
+# downloads bar.jpg and stores it in the current directory
+```
+
+or
+
+```
+nugget http://foo.com/bar.jpg -O baz.jpg
+# saves it as baz.jpg. you can also do lowercase -o
+```
+
+if you get a statusCode of 300 or greater nugget will stop. you can force it to stream the response into a file anyway by doing `nugget http://404link.com/file.html -f` or `--force` works too
+
+you can also download multiple files, just pass multiple urls:
+
+![download multiple](multiple.png)
+
+## options
+
+The following options are recognized by nugget:
+
+ * `-s|--sockets` - default Infinity. specify the number of http sockets to use at once (this controls concurrency)
+ * `-o|-O|--out` - specify the filename to write to. this only works if you are downloading a single file
+ * `-d|--dir` - save files in a directory other than the current one.
+ * `-c|--continue` - resume downloads if a partially complete target file already exists. If the target file exists and is the same size as the remote file, nothing will be done.
+ * `-f|--force` - force the server response to be saved to the target file, even if it's a non-successful status code.
+ * `--no-strict-ssl` - disable strict ssl
+ * `--proxy` - specify a proxy to use

node_modules/nugget/test/resume.js

@@ -0,0 +1,40 @@
+var fs = require('fs')
+var http = require('http')
+var nugget = require('../')
+var path = require('path')
+var test = require('tape')
+
+var data = new Buffer('hello everybody I am the data');
+
+var testServer = http.createServer(function(req, res) {
+  if (!req.headers['range']) {
+    res.setHeader('content-length', data.length)
+    res.setHeader('accept-ranges', 'bytes')
+    res.end(data)
+  } else {
+    var range = req.headers['range'].split('=').pop().split('-').map(function (s) {
+      return parseInt(s, 10)
+    })
+    res.setHeader('content-length', range[1] - range[0])
+    res.setHeader('content-range', range[0] + '-' + range[1] + '/' + data.length);
+    res.end(data.slice(range[0], range[1]))
+  }
+})
+
+var target = path.join(__dirname, 'foobar.html')
+if (fs.existsSync(target)) fs.unlinkSync(target)
+
+fs.writeFileSync(target, data.slice(0, 10))
+
+testServer.listen(0, function() {
+  var port = this.address().port
+  test('fetches rest of file', function(t) {
+    nugget('http://localhost:' + port + '/foobar.html', {dir: __dirname, resume: true}, function(err) {
+      t.ok(fs.existsSync(target), 'downloaded file')
+      t.equal(fs.statSync(target).size, data.length, 'file is complete');
+      if (fs.existsSync(target)) fs.unlinkSync(target)
+      t.end()
+      testServer.close()
+    })
+  })
+})

node_modules/nugget/test/test.js

@@ -0,0 +1,37 @@
+var fs = require('fs')
+var http = require('http')
+var nugget = require('../')
+var path = require('path')
+var test = require('tape')
+
+var testServer = http.createServer(function (req, res) {
+  res.end('hello')
+})
+
+var target = path.join(__dirname, 'resume.html')
+if (fs.existsSync(target)) fs.unlinkSync(target)
+
+testServer.listen(0, function () {
+  var port = this.address().port
+  test('fetches file', function (t) {
+    nugget('http://localhost:' + port + '/resume.html', {dir: __dirname}, function (err) {
+      t.ok(fs.existsSync(target), 'downloaded file')
+      if (fs.existsSync(target)) fs.unlinkSync(target)
+      t.end()
+    })
+  })
+  
+  test('has progress events', function (t) {
+    var gotProgress = false
+    var dl = nugget('http://localhost:' + port + '/resume.html', {dir: __dirname}, function (err) {
+      t.notOk(err, 'no error')
+      t.ok(gotProgress, 'got progress event')
+      t.end()
+      testServer.close()
+    })
+    dl.once('progress', function (data) {
+      t.ok(data.hasOwnProperty('percentage'), 'has percentage')
+      gotProgress = true
+    })
+  })
+})