Merge pull request #1192 from aboodman/fs

Introduce clients/fs: beginning of a filesystem importer

clients/fs/.flowconfig

@@ -0,0 +1 @@
+../../js/.flowconfig

clients/fs/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+out.js

clients/fs/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "directory-import",
+  "dependencies": {
+    "@attic/noms": "^9.0.1"
+  },
+  "devDependencies": {
+    "babel-cli": "6.6.5",
+    "babel-core": "6.7.2",
+    "babel-eslint": "5.0.0",
+    "babel-generator": "6.7.2",
+    "babel-plugin-syntax-async-functions": "6.5.0",
+    "babel-plugin-syntax-flow": "6.5.0",
+    "babel-plugin-transform-async-to-generator": "6.7.0",
+    "babel-plugin-transform-class-properties": "6.6.0",
+    "babel-plugin-transform-es2015-destructuring": "6.6.5",
+    "babel-plugin-transform-es2015-modules-commonjs": "6.7.0",
+    "babel-plugin-transform-es2015-parameters": "6.7.0",
+    "babel-plugin-transform-runtime": "^6.6.0",
+    "babel-preset-es2015": "6.6.0",
+    "babel-preset-react": "6.5.0",
+    "babel-regenerator-runtime": "6.5.0",
+    "eslint-plugin-flow-vars": "^0.2.1",
+    "eslint": "^1.9.0",
+    "eslint-plugin-react": "^4.3.0",
+    "flow-bin": "^0.22.1",
+    "oauth": "^0.9.14",
+    "mz": "2.4.0",
+    "yargs": "4.4.0"
+  },
+  "scripts": {
+    "start": "babel -d dist -w src",
+    "build": "babel -d dist src",
+    "test": "eslint src/ && flow src/"
+  }
+}

clients/fs/src/.babelrc

@@ -0,0 +1 @@
+../../../js/.babelrc

clients/fs/src/.eslintrc

@@ -0,0 +1,47 @@
+{
+  "parser": "babel-eslint",
+  "rules": {
+    "array-bracket-spacing": [2, "never"],
+    "arrow-body-style": [2, "as-needed"],
+    "arrow-parens": 0,
+    "arrow-spacing": [2, { "before": true, "after": true }],
+    "camelcase": 2,
+    "comma-dangle": [2, "always-multiline"],
+    "eqeqeq": 2,
+    "flow-vars/define-flow-type": 1,
+    "flow-vars/use-flow-type": 1,
+    "indent": [2, 2, {"SwitchCase": 1}],
+    "linebreak-style": [2, "unix"],
+    "max-len": [2, 100, 8],
+    "no-console": 0,
+    "no-fallthrough": 2,
+    "no-multi-spaces": 2,
+    "no-new-wrappers": 2,
+    "no-throw-literal": 2,
+    "no-var": 2,
+    "object-curly-spacing": [2, "never"],
+    "prefer-arrow-callback": 2,
+    "prefer-const": 2,
+    "quotes": [2, "single"],
+    "radix": 2,
+    "semi": 2,
+    "space-after-keywords": 2,
+    "space-before-function-paren": 0,
+    "space-in-parens": [2, "never"],
+    "space-infix-ops": 2,
+  },
+  "env": {
+    "es6": true,
+    "node": true,
+    "browser": true
+  },
+  "extends": "eslint:recommended",
+  "ecmaFeatures": {
+    "jsx": true,
+    "experimentalObjectRestSpread": true
+  },
+  "plugins": [
+    "flow-vars",
+    "react"
+  ]
+}

clients/fs/src/fs.noms

@@ -0,0 +1,16 @@
+using Map<String, Ref<DirectoryEntry>>
+
+struct Directory {
+	entries: Map<String, Ref<DirectoryEntry>>
+}
+
+struct File {
+	content: Ref<Blob>
+}
+
+struct DirectoryEntry {
+	union {
+		file: File
+		directory: Directory
+	}
+}

clients/fs/src/fs.noms.js

@@ -0,0 +1,103 @@
+// This file was generated by nomdl/codegen.
+// @flow
+/* eslint-disable */
+
+import {
+  Field as _Field,
+  Kind as _Kind,
+  Package as _Package,
+  Ref as _Ref,
+  blobType as _blobType,
+  createStructClass as _createStructClass,
+  makeCompoundType as _makeCompoundType,
+  makeMapType as _makeMapType,
+  makeStructType as _makeStructType,
+  makeType as _makeType,
+  newMap as _newMap,
+  registerPackage as _registerPackage,
+  stringType as _stringType,
+} from '@attic/noms';
+import type {
+  Blob as _Blob,
+  NomsMap as _NomsMap,
+  RefValue as _RefValue,
+  Struct as _Struct,
+} from '@attic/noms';
+
+const _pkg = new _Package([
+  _makeStructType('Directory',
+      [
+        new _Field('entries', _makeCompoundType(_Kind.Map, _stringType, _makeCompoundType(_Kind.Ref, _makeType(new _Ref(), 2))), false),
+      ],
+      [
+
+      ]
+    ),
+  _makeStructType('File',
+      [
+        new _Field('content', _makeCompoundType(_Kind.Ref, _blobType), false),
+      ],
+      [
+
+      ]
+    ),
+  _makeStructType('DirectoryEntry',
+      [
+
+      ],
+      [
+        new _Field('file', _makeType(new _Ref(), 1), false),
+        new _Field('directory', _makeType(new _Ref(), 0), false),
+      ]
+    ),
+], [
+]);
+_registerPackage(_pkg);
+export const typeForDirectory = _makeType(_pkg.ref, 0);
+const Directory$typeDef = _pkg.types[0];
+export const typeForFile = _makeType(_pkg.ref, 1);
+const File$typeDef = _pkg.types[1];
+export const typeForDirectoryEntry = _makeType(_pkg.ref, 2);
+const DirectoryEntry$typeDef = _pkg.types[2];
+
+
+type Directory$Data = {
+  entries: _NomsMap<string, _RefValue<DirectoryEntry>>;
+};
+
+interface Directory$Interface extends _Struct {
+  constructor(data: Directory$Data): void;
+  entries: _NomsMap<string, _RefValue<DirectoryEntry>>;  // readonly
+  setEntries(value: _NomsMap<string, _RefValue<DirectoryEntry>>): Directory$Interface;
+}
+
+export const Directory: Class<Directory$Interface> = _createStructClass(typeForDirectory, Directory$typeDef);
+
+type File$Data = {
+  content: _RefValue<_Blob>;
+};
+
+interface File$Interface extends _Struct {
+  constructor(data: File$Data): void;
+  content: _RefValue<_Blob>;  // readonly
+  setContent(value: _RefValue<_Blob>): File$Interface;
+}
+
+export const File: Class<File$Interface> = _createStructClass(typeForFile, File$typeDef);
+
+type DirectoryEntry$Data = {
+};
+
+interface DirectoryEntry$Interface extends _Struct {
+  constructor(data: DirectoryEntry$Data): void;
+  file: ?File;  // readonly
+  setFile(value: File): DirectoryEntry$Interface;
+  directory: ?Directory;  // readonly
+  setDirectory(value: Directory): DirectoryEntry$Interface;
+}
+
+export const DirectoryEntry: Class<DirectoryEntry$Interface> = _createStructClass(typeForDirectoryEntry, DirectoryEntry$typeDef);
+
+export function newMapOfStringToRefOfDirectoryEntry(values: Array<any>): Promise<_NomsMap<string, _RefValue<DirectoryEntry>>> {
+  return _newMap(values, _makeMapType(_stringType, _makeCompoundType(_Kind.Ref, _makeType(_pkg.ref, 2))));
+}

clients/fs/src/main.js

@@ -0,0 +1,138 @@
+// @flow
+
+import fs from 'mz/fs';
+import path from 'path';
+import argv from 'yargs';
+import {
+  newMapOfStringToRefOfDirectoryEntry,
+  Directory,
+  DirectoryEntry,
+  File,
+} from './fs.noms.js';
+import {
+  BlobWriter,
+  Dataset,
+  DataStore,
+  HttpStore,
+  NomsBlob,
+  RefValue,
+} from '@attic/noms';
+
+const args = argv
+  .usage('Usage: $0 <path> <dataset>')
+  .command('path', 'filesystem path to import')
+  .demand(1)
+  .command('dataset', 'dataset to write to')
+  .demand(1)
+  .argv;
+
+let numFilesFound = 0;
+let numFilesComplete = 0;
+
+main().catch(ex => {
+  console.error(ex.stack);
+  process.exit(1);
+});
+
+async function main(): Promise<void> {
+  const [p, datastoreSpec, datasetName] = parseArgs();
+  if (!p) {
+    process.exit(1);
+    return;
+  }
+
+  const store = new DataStore(new HttpStore(datastoreSpec));
+  const ds = new Dataset(store, datasetName);
+
+  const r = await processPath(p, store);
+  if (r) {
+    await ds.commit(r);
+    process.stdout.write('\ndone\n');
+  }
+}
+
+async function processPath(p: string, store: DataStore): Promise<?RefValue<DirectoryEntry>> {
+  numFilesFound++;
+  const st = await fs.stat(p);
+  let de = null;
+  if (st.isDirectory()) {
+    de = new DirectoryEntry({
+      directory: await processDirectory(p, store),
+    });
+  } else if (st.isFile()) {
+    de = new DirectoryEntry({
+      file: await processFile(p, store),
+    });
+  } else {
+    console.info('Skipping path %s because this filesystem node type is not currently handled', p);
+    return null;
+  }
+
+  return await store.writeValue(de);
+}
+
+async function processDirectory(p: string, store: DataStore): Promise<Directory> {
+  const names = await fs.readdir(p);
+  const children = names.map(name => {
+    const chPath = path.join(p, name);
+    return processPath(chPath, store).then(dirEntryRef => [chPath, dirEntryRef]);
+  });
+
+  numFilesComplete++;
+  updateProgress();
+
+  const resolved = await Promise.all(children);
+  const entries = resolved
+    .filter(([, dirEntryRef]) => dirEntryRef)
+    .reduce((l, t) => { l.push(...t); return l; }, []);
+  const fm = await newMapOfStringToRefOfDirectoryEntry(entries);
+  return new Directory({
+    entries: fm,
+  });
+}
+
+async function processFile(p: string, store: DataStore): Promise<File> {
+  const f = new File({
+    content: await processBlob(p, store),
+  });
+  numFilesComplete++;
+  updateProgress();
+  return f;
+}
+
+function processBlob(p: string, store: DataStore): Promise<RefValue<NomsBlob>> {
+  const w = new BlobWriter();
+  const s = fs.createReadStream(p);
+  return new Promise((res, rej) => {
+    s.on('data', chunk => w.write(chunk));
+    s.on('end', async () => {
+      await w.close();
+      try {
+        res(store.writeValue(w.blob));
+      } catch (ex) {
+        rej(ex);
+      }
+    });
+    s.on('error', rej);
+  });
+}
+
+function updateProgress() {
+  process.stdout.write(`\r${numFilesComplete} of ${numFilesFound} entries processed...`);
+}
+
+function parseArgs() {
+  const [p, datasetSpec] = args._;
+  const parts = datasetSpec.split(':');
+  if (parts.length < 2) {
+    console.error('invalid dataset spec');
+    return [];
+  }
+  const datasetName = parts.pop();
+  const datastoreSpec = parts.join(':');
+  if (!/^http/.test(datastoreSpec)) {
+    console.error('Unsupported datastore type: ', datastoreSpec);
+    return [];
+  }
+  return [p, datastoreSpec, datasetName];
+}