Update flickr to newest version of noms sdk (#1602)

clients/js/flickr/find-photos/package.json

@@ -3,7 +3,7 @@
   "description": "Creates standard photo structs out of slurped flickr data",
   "main": "dist/main.js",
   "dependencies": {
-    "@attic/noms": "^32.2.0",
+    "@attic/noms": "39.2.0",
     "babel-regenerator-runtime": "6.5.0",
     "humanize": "0.0.9",
     "node-fetch": "1.5.2",

clients/js/flickr/find-photos/src/main.js

@@ -3,15 +3,17 @@
 import argv from 'yargs';
 import {
   DatasetSpec,
-  invariant,
-  newMap,
-  newSet,
+  isSubtype,
+  makeStructType,
+  makeUnionType,
+  Map,
   newStruct,
+  numberType,
+  Set,
+  stringType,
   Struct,
-  StructMirror,
   walk,
 } from '@attic/noms';
-import type {Map} from '@attic/noms';
 
 const args = argv
   .usage(
@@ -20,6 +22,36 @@ const args = argv
   .demand(2)
   .argv;
 
+const sizes = ['t', 's', 'm', 'l', 'o'];
+const flickrNum = makeUnionType([stringType, numberType]);
+const sizeTypes = sizes.map(s =>
+  makeStructType('', {
+    ['url_' + s]: stringType,
+    ['width_' + s]: flickrNum,
+    ['height_' + s]: flickrNum,
+  }));
+
+// This is effectively:
+// union {
+//   struct {
+//     title: string,
+//     tags: string,
+//     latitude: flickrNum,
+//     longitude: flickrNum,
+//     url_t: string,
+//     width_t: flickrNum,
+//     height_t: flickrNum,
+//   } |
+//   ... for all the image size suffixes ...
+// }
+const imageType = makeUnionType(sizeTypes.map(st =>
+    makeStructType('', Object.assign(({
+      title: stringType,
+      tags: stringType,
+      latitude: flickrNum,
+      longitude: flickrNum,
+    }:Object), st.desc.fields))));
+
 main().catch(ex => {
   console.error(ex);
   process.exit(1);
@@ -36,51 +68,45 @@ async function main(): Promise<void> {
   }
 
   const input = await inSpec.value();
-  const output = outSpec.set();
-  let result = newSet([]);
+  const output = outSpec.dataset();
+  let result = Promise.resolve(new Set());
 
   // TODO: How to report progress?
-  await walk(input, output.store, async v => {
-    // TODO: Use some kind of subtype/instanceof check instead.
-    if (v instanceof Struct && v.url_t) {
+  await walk(input, output.database, v => {
+    if (v instanceof Struct && isSubtype(imageType, v.type)) {
       const s = newStruct('Photo', {
         title: v.title || '',
-        tags: await newSet(v.tags ? v.tags.split(' ') : []),
+        tags: new Set(v.tags ? v.tags.split(' ') : []),
         geoposition: getGeo(v),
-        sizes: await getSizes(v),
+        sizes: getSizes(v),
       });
       result = result.then(r => r.insert(s));
-      return false;
+      return true;
     }
-    return true;
+    return false;
   });
 
   return output.commit(await result).then();
 }
 
-function getGeo(input: Struct): Struct {
+function getGeo(input: Object): Struct {
   const geopos = {
-    latitude: input.latitude || 0,
-    longitude: input.longitude || 0,
+    latitude: Number(input.latitude || 0),
+    longitude: Number(input.longitude || 0),
   };
   return newStruct('Geoposition', geopos);
 }
 
-function getSizes(input: Struct): Promise<Map<Struct, string>> {
-  let res: Promise<Map<Struct, string>> = newMap([]);
-
-  // TODO: Really want to do Go-style interface checking here.
-  // Could have one struct for each size, then just check each one in turn, add it if present.
-  const mirror = new StructMirror(input);
-  ['t', 's', 'm', 'l', 'o'].forEach(tag => {
-    const url = mirror.get('url_' + tag);
-    if (url) {
-      invariant(typeof url === 'string');
-      const width = Number(mirror.get('width_' + tag));
-      const height = Number(mirror.get('height_' + tag));
-      res = res.then(r => r.set(newStruct('', {width, height}), url));
-    }
-  });
-
-  return res;
+function getSizes(input: Object): Map<Struct, string> {
+  return new Map(
+    sizes.map((s, i) => {
+      if (!isSubtype(sizeTypes[i], input.type)) {
+        // $FlowIssue - Flow doesn't realize that filter will return only non-nulls.
+        return null;
+      }
+      const url = input['url_' + s];
+      const width = Number(input['width_' + s]);
+      const height = Number(input['height_' + s]);
+      return [newStruct('', {width, height}), url];
+    }).filter(kv => kv));
 }

clients/js/flickr/slurp/package.json

@@ -3,7 +3,7 @@
   "description": "Imports album and photo metadata from Flickr",
   "main": "dist/main.js",
   "dependencies": {
-    "@attic/noms": "^32.2.0",
+    "@attic/noms": "39.2.0",
     "babel-regenerator-runtime": "6.5.0",
     "flickr-oauth-and-upload": "0.8.0",
     "humanize": "0.0.9",

clients/js/flickr/slurp/src/main.js

@@ -7,14 +7,11 @@ import {
   Dataset,
   DatasetSpec,
   invariant,
-  newList,
-  newSet,
+  jsonToNoms,
   newStruct,
+  Set,
   Struct,
 } from '@attic/noms';
-import type {
-  valueOrPrimitive,
-} from '@attic/noms';
 
 const args = argv
   .usage(
@@ -62,7 +59,7 @@ async function main(): Promise<void> {
     throw 'invalid destination dataset spec';
   }
 
-  out = outSpec.set();
+  out = outSpec.dataset();
 
   if (args['auth-token'] && args['auth-secret']) {
     authToken = args['auth-token'];
@@ -81,11 +78,11 @@ async function main(): Promise<void> {
         `${clearLine}${++seen} of ${photosetsJSON.length} photosets imported...`);
       return p;
     });
-  })).then(sets => newSet(sets));
+  })).then(sets => new Set(sets));
 
   process.stdout.write(clearLine);
   return out.commit(newStruct('', {
-    photosetsMeta: await toNoms(photosetsJSON),
+    photosetsMeta: jsonToNoms(photosetsJSON),
     photosets: await photosets,
   })).then();
 }
@@ -101,7 +98,7 @@ async function getPhotoset(id: string): Promise<Struct> {
       'last_update, geo, tags, machine_tags, o_dims, views, media, path_alias, url_sq, url_t, ' +
       'url_s, url_m, url_o',
   });
-  const res = await toNoms(json.photoset);
+  const res = jsonToNoms(json.photoset);
   invariant(res instanceof Struct);
   return res;
 }
@@ -134,7 +131,8 @@ function promptForAuth(url: string): Promise<void> {
     process.stdout.write(`Go to ${url} to grant permissions to access Flickr...\n`);
     const rl = readline.createInterface({input: process.stdin, output: process.stdout});
     rl.question('Press enter when done\n', () => {
-      process.stdout.write(`Authenticated: authToken: ${authToken} - authSecret: ${authSecret}\n`);
+      process.stdout.write('Authenticated. Next time run:\n' +
+        `${process.argv.join(' ')} --auth-token=${authToken} --auth-secret=${authSecret}\n\n`);
       res();
       rl.close();
     });
@@ -160,27 +158,3 @@ function callFlickr(method: string, params: ?{[key: string]: string}) {
     });
   });
 }
-
-async function toNoms(v: any): Promise<valueOrPrimitive> {
-  switch (typeof v) {
-    case 'boolean':
-    case 'number':
-    case 'string':
-      return v;
-  }
-
-  if (v instanceof Array) {
-    return await newList(await Promise.all(v.map(c => toNoms(c))));
-  }
-
-  if (v instanceof Object) {
-    const props = {};
-    await Promise.all(
-      Object.keys(v).map(k => {
-        return toNoms(v[k]).then(c => props[k] = c);
-      }));
-    return newStruct('', props);
-  }
-
-  throw new Error('unexpected type: ' + v);
-}