Add lost clients/js/url_fetch (#1509)

This was accidentally removed in the clients dir reshuffle a few days back.

clients/js/url_fetch/.flowconfig

@@ -0,0 +1 @@
+../../../js/.flowconfig

clients/js/url_fetch/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

clients/js/url_fetch/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "noms-url-fetch",
+  "main": "dist/main.js",
+  "dependencies": {
+    "@attic/noms": "^26.0.0",
+    "babel-regenerator-runtime": "6.5.0",
+    "humanize": "0.0.9",
+    "yargs": "4.4.0"
+  },
+  "devDependencies": {
+    "@attic/eslintrc": "^1.0.0",
+    "babel-cli": "6.6.5",
+    "babel-core": "6.7.2",
+    "babel-generator": "6.7.2",
+    "babel-plugin-syntax-async-functions": "6.5.0",
+    "babel-plugin-syntax-flow": "6.5.0",
+    "babel-plugin-transform-async-to-generator": "6.7.0",
+    "babel-plugin-transform-class-properties": "6.6.0",
+    "babel-plugin-transform-es2015-destructuring": "6.6.5",
+    "babel-plugin-transform-es2015-modules-commonjs": "6.7.0",
+    "babel-plugin-transform-es2015-parameters": "6.7.0",
+    "babel-plugin-transform-runtime": "^6.6.0",
+    "babel-preset-es2015": "6.6.0",
+    "babel-preset-react": "6.5.0",
+    "chai": "3.5.0",
+    "flow-bin": "^0.23.0",
+    "mocha": "2.4.5"
+  },
+  "scripts": {
+    "start": "babel -d dist -w src",
+    "build": "babel -d dist src",
+    "test": "eslint src/ && flow src/"
+  }
+}

clients/js/url_fetch/src/.babelrc

@@ -0,0 +1 @@
+../../../../js/.babelrc

clients/js/url_fetch/src/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = require('@attic/eslintrc');
+// Allow console
+module.exports.rules['no-console'] = 0;

clients/js/url_fetch/src/main.js

@@ -0,0 +1,96 @@
+// @flow
+
+import argv from 'yargs';
+import http from 'http';
+import humanize from 'humanize';
+import {
+  BlobWriter,
+  DatasetSpec,
+  invariant,
+  NomsBlob,
+} from '@attic/noms';
+
+const args = argv
+  .usage('Usage: $0 <url> <dataset>')
+  .command('url', 'url to import')
+  .command('dataset', 'dataset spec to write to')
+  .demand(2)
+  .argv;
+
+const clearLine = '\x1b[2K\r';
+const startTime = Date.now();
+
+let expectedBytes = 0;
+let expectedBytesHuman = '';
+let completedBytes = 0;
+
+main().catch(ex => {
+  console.error(ex.stack);
+  process.exit(1);
+});
+
+function main(): Promise<void> {
+  const url = args._[0];
+  const spec = DatasetSpec.parse(args._[1]);
+  if (!spec) {
+    process.stderr.write('invalid dataset spec');
+    process.exit(1);
+    return Promise.resolve();
+  }
+
+  const set = spec.set();
+  return getBlob(url)
+    .then(b => set.commit(b))
+    .then(() => {
+      process.stderr.write(clearLine + 'Done\n');
+    });
+}
+
+function getBlob(url): Promise<NomsBlob> {
+  const w = new BlobWriter();
+
+  return new Promise(resolve => {
+    http.get(url, res => {
+      switch (Math.floor(res.statusCode / 100)) {
+        case 4:
+        case 5:
+          invariant(res.statusMessage);
+          process.stderr.write(`Error fetching ${url}: ${res.statusCode}: ${res.statusMessage}\n`);
+          process.exit(1);
+          break;
+      }
+
+      process.stdout.write(clearLine + `got ${res.statusCode}, continuing...\n`);
+
+      const header = res.headers['content-length'];
+      if (header) {
+        expectedBytes = Number(header);
+        expectedBytesHuman = humanize.filesize(expectedBytes);
+      } else {
+        expectedBytesHuman = '(unknown)';
+      }
+
+      res.on('error', e => {
+        process.stderr.write(`Error fetching ${url}: ${e.message}`);
+        process.exit(1);
+      });
+
+      res.on('data', chunk => {
+        w.write(chunk);
+        completedBytes += chunk.length;
+        const elapsed = (Date.now() - startTime) / 1000;
+        const rate = humanize.filesize(completedBytes / elapsed);
+        process.stdout.write(clearLine + `${humanize.filesize(completedBytes)} of ` + 
+            `${expectedBytesHuman} written in ${elapsed}s (${rate}/s)`);
+      });
+
+      res.on('end', () => {
+        process.stdout.write(clearLine + 'Committing...');
+        w.close()
+          .then(() => resolve(w.blob));
+      });
+
+      res.resume();
+    });
+  });
+}