Merge pull request #96 from todvora/plugin-infrastructure

Plugin infrastructure

.jshintrc

@@ -13,13 +13,16 @@
     "BinData": false,
     "DBQuery": false,
     "printjson": false,
+    "load": false,
+    "module": false,
 
     "query": true,
     "limit": true,
     "maxDepth": true,
     "sort": true,
     "outputFormat": true,
-    "persistResults": true
+    "persistResults": true,
+    "plugins": true
   },
   "latedef": true,
   "singleGroups": true,

CHANGELOG

+(14 May 2015) Version 1.5.0: Introduced basic plugin infrastructure, 'onConfig' and 'formatResults' hooks
+
 (14 Oct 2014) Version 1.4.1: @todvora's fix for maxDepth (matches readme), and fixes for incorrect counts while using query/limit
 
 (13 Oct 2014) Version 1.4.0: @todvora's fix for nested objects

test/src/main/java/com/github/variety/Variety.java

@@ -32,6 +32,8 @@ public class Variety {
     public static final String PARAM_LIMIT = "limit";
     public static final String PARAM_OUTPUT_FORMAT = "outputFormat";
     public static final String PARAM_PERSIST_RESULTS = "persistResults";
+    public static final String PARAM_PLUGINS = "plugins";
+
 
     private final String inputDatabase;
     private final String inputCollection;
@@ -46,6 +48,7 @@ public class Variety {
     private String outputFormat;
     private boolean quiet;
     private boolean persistResults;
+    private String[] plugins;
 
     /**
      * Create variety wrapper with defined connection do analysed database and collection
@@ -141,6 +144,11 @@ public class Variety {
         return this;
     }
 
+    public Variety withPlugins(String... plugins) {
+        this.plugins = plugins;
+        return this;
+    }
+
     /**
      * Executes mongo shell with configured variety options and variety.js script in path.
      * @return Stdout of variety.js
@@ -192,6 +200,11 @@ public class Variety {
         if(persistResults) {
             args.add(PARAM_PERSIST_RESULTS + " = " + persistResults);
         }
+
+        if(plugins != null && plugins.length > 0) {
+            args.add(PARAM_PLUGINS + " = \"" + String.join(",", plugins) +  "\"");
+        }
+
         return args.toString();
     }
 

test/src/test/java/com/github/variety/test/ParametersParsingTest.java

@@ -83,14 +83,14 @@ public class ParametersParsingTest {
     public void testDefaultOutputFormatParam() throws Exception {
         final ResultsValidator analysis = variety.runDatabaseAnalysis(); // format option not provided
         final Map<String, String> params = getParamsMap(analysis.getStdOut());
-        Assert.assertEquals("ascii", params.get(Variety.PARAM_OUTPUT_FORMAT));
+        Assert.assertEquals("\"ascii\"", params.get(Variety.PARAM_OUTPUT_FORMAT));
     }
 
     @Test
     public void testAsciiOutputFormatParam() throws Exception {
         final ResultsValidator analysis = variety.withFormat(Variety.FORMAT_ASCII).runDatabaseAnalysis();
         final Map<String, String> params = getParamsMap(analysis.getStdOut());
-        Assert.assertEquals("ascii", params.get(Variety.PARAM_OUTPUT_FORMAT));
+        Assert.assertEquals("\"ascii\"", params.get(Variety.PARAM_OUTPUT_FORMAT));
     }
 
     @Test

test/src/test/java/com/github/variety/test/PluginsTest.java

+package com.github.variety.test;
+
+import com.github.variety.Variety;
+import com.github.variety.validator.ResultsValidator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URL;
+
+public class PluginsTest {
+
+    private Variety variety;
+
+    public static final String EXPECTED_OUTPUT =
+            "key|types|occurrences|percents\n" +
+            "_id|ObjectId|5|100\n" +
+            "name|String|5|100\n" +
+            "bio|String|3|60\n" +
+            "birthday|String|2|40\n" +
+            "pets|Array,String|2|40\n" +
+            "someBinData|BinData-old|1|20\n" +
+            "someWeirdLegacyKey|String|1|20";
+
+    @Before
+    public void setUp() throws Exception {
+        this.variety = new Variety("test", "users");
+        variety.getSourceCollection().insert(SampleData.getDocuments());
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        variety.getVarietyResultsDatabase().dropDatabase();
+        variety.getSourceCollection().drop();
+    }
+
+    /**
+     * Validate correct results read from DB
+     */
+    @Test
+    public void verifyFormatResults() throws Exception {
+        final String path = getPluginPath("/csvplugin.js");
+        final ResultsValidator analysis = variety.withQuiet(true).withPlugins(path).runDatabaseAnalysis();
+        Assert.assertEquals(EXPECTED_OUTPUT, analysis.getStdOut());
+    }
+
+    @Test
+    public void verifyPluginParamParsing() throws Exception {
+        final String path = getPluginPath("/csvplugin.js");
+        final ResultsValidator analysis = variety.withPlugins(path + "|delimiter=;").runDatabaseAnalysis();
+        Assert.assertTrue(analysis.getStdOut().contains("Using plugins of [ \"" + path + "\" ]"));
+    }
+
+    private String getPluginPath(final String name) {
+        final URL resource = this.getClass().getResource(name);
+        return resource.getFile();
+    }
+}

test/src/test/resources/csvplugin.js

+var getCsv = function(varietyResults) {
+  var delimiter = this.delimiter || '|';
+  var headers = ['key', 'types', 'occurrences', 'percents'];
+  var table = [headers.join(delimiter)];
+  var rows = varietyResults.map(function(key) {
+    return [key._id.key, key.value.types, key.totalOccurrences, key.percentContaining].join(delimiter);
+  }, this);
+  return table.concat(rows).join('\n');
+};
+
+var setConfig = function(pluginConfig) {
+  this.delimiter = pluginConfig.delimiter;
+};
+
+module.exports = {
+  init: setConfig,
+  formatResults: getCsv
+};

variety.js

@@ -18,7 +18,7 @@ var log = function(message) {
 };
 
 log('Variety: A MongoDB Schema Analyzer');
-log('Version 1.4.1, released 14 Oct 2014');
+log('Version 1.5.0, released 14 May 2015');
 
 var dbs = [];
 var emptyDbs = [];
@@ -57,49 +57,69 @@ if (db[collection].count() === 0) {
         'Possible collection options for database specified: ' + collNames + '.';
 }
 
-var $query = {};
-if(typeof query !== 'undefined') {
-  $query = query;
-  query = '_undefined';
-}
-log('Using query of ' + tojson($query));
-
-var $limit = db[collection].find($query).count();
-if(typeof limit !== 'undefined') {
-  $limit = limit;
-  limit = '_undefined';
-}
-log('Using limit of ' + $limit);
+var readConfig = function(configProvider) {
+  var config = {};
+  var read = function(name, defaultValue) {
+    var value = typeof configProvider[name] !== 'undefined' ? configProvider[name] : defaultValue;
+    config[name] = value;
+    log('Using '+name+' of ' + tojson(value));
+  };
+  read('collection', null);
+  read('query', {});
+  read('limit', db[config.collection].find(config.query).count());
+  read('maxDepth', 99);
+  read('sort', {_id: -1});
+  read('outputFormat', 'ascii');
+  read('persistResults', false);
+  return config;
+};
 
-var $maxDepth = 99;
-if(typeof maxDepth !== 'undefined') {
-  $maxDepth = maxDepth;
-  maxDepth = '_undefined';
-}
-log('Using maxDepth of ' + $maxDepth);
+var config = readConfig(this);
+
+var PluginsClass = function(context) {
+  var parsePath = function(val) { return val.slice(-3) !== '.js' ? val + '.js' : val;};
+  var parseConfig = function(val) {
+    var config = {};
+    val.split('&').reduce(function(acc, val) {
+      var parts = val.split('=');
+      acc[parts[0]] = parts[1];
+      return acc;
+    }, config);
+    return config;
+  };
 
-var $sort = {_id: -1};
-if(typeof sort !== 'undefined') {
-  $sort = sort;
-  sort = '_undefined';
-}
-log('Using sort of ' + tojson($sort));
+  if(typeof context.plugins !== 'undefined') {
+    this.plugins = context.plugins.split(',')
+      .map(function(path){return path.trim();})
+      .map(function(definition){
+        var path = parsePath(definition.split('|')[0]);
+        var config = parseConfig(definition.split('|')[1] || '');
+        context.module = context.module || {};
+        load(path);
+        var plugin = context.module.exports;
+        plugin.path = path;
+        if(typeof plugin.init === 'function') {
+          plugin.init(config);
+        }
+        return plugin;
+      }, this);
+  } else {
+    this.plugins = [];
+  }
 
-var $outputFormat = 'ascii';
-if(typeof outputFormat !== 'undefined') {
-  $outputFormat = outputFormat;
-  outputFormat = '_undefined';
-}
-log('Using outputFormat of ' + $outputFormat);
+  this.execute = function(methodName) {
+    var args = Array.prototype.slice.call(arguments, 1);
+    var applicablePlugins = this.plugins.filter(function(plugin){return typeof plugin[methodName] === 'function';});
+    return applicablePlugins.map(function(plugin) {
+      return plugin[methodName].apply(plugin, args);
+    });
+  };
 
-var $persistResults = false;
-if(typeof persistResults !== 'undefined') {
-  $persistResults = persistResults;
-  persistResults = '_undefined';
-}
-log('Using persistResults of ' + $persistResults);
+  log('Using plugins of ' + tojson(this.plugins.map(function(plugin){return plugin.path;})));
+};
 
-log('Using collection of ' + collection);
+var $plugins = new PluginsClass(this);
+$plugins.execute('onConfig', config);
 
 var varietyTypeOf = function(thing) {
   if (typeof thing === 'undefined') { throw 'varietyTypeOf() requires an argument'; }
@@ -228,7 +248,7 @@ var convertResults = function(interimResults, documentsCount) {
 
 // Merge the keys and types of current object into accumulator object
 var reduceDocuments = function(accumulator, object) {
-  var docResult = analyseDocument(serializeDoc(object, $maxDepth));
+  var docResult = analyseDocument(serializeDoc(object, config.maxDepth));
   mergeDocument(docResult, accumulator);
   return accumulator;
 };
@@ -254,13 +274,13 @@ DBQuery.prototype.reduce = function(callback, initialValue) {
   return result;
 };
 
-var cursor = db[collection].find($query).sort($sort).limit($limit);
+var cursor = db[config.collection].find(config.query).sort(config.sort).limit(config.limit);
 var interimResults = cursor.reduce(reduceDocuments, {});
 var varietyResults = convertResults(interimResults, cursor.size())
   .filter(filter)
   .sort(comparator);
 
-if($persistResults) {
+if(config.persistResults) {
   var resultsDB = db.getMongo().getDB('varietyResults');
   var resultsCollectionName = collection + 'Keys';
 
@@ -270,36 +290,36 @@ if($persistResults) {
   resultsDB[resultsCollectionName].insert(varietyResults);
 }
 
-if($outputFormat === 'json') {
-  printjson(varietyResults); // valid formatted json output, compressed variant is printjsononeline()
-} else {  // output nice ascii table with results
-  var table = [['key', 'types', 'occurrences', 'percents'], ['', '', '', '']]; // header + delimiter rows
-
-   // return the number of decimal places or 1, if the number is int (1.23=>2, 100=>1, 0.1415=>4)
-   var significantDigits = function(value) {
-      var res = value.toString().match(/^[0-9]+\.([0-9]+)$/);
-      return res !== null ? res[1].length : 1;
-    };
+var createAsciiTable = function(results) {
+  var headers = ['key', 'types', 'occurrences', 'percents'];
+  // return the number of decimal places or 1, if the number is int (1.23=>2, 100=>1, 0.1415=>4)
+  var significantDigits = function(value) {
+    var res = value.toString().match(/^[0-9]+\.([0-9]+)$/);
+    return res !== null ? res[1].length : 1;
+  };
 
   var maxDigits = varietyResults.map(function(value){return significantDigits(value.percentContaining);}).reduce(function(acc,val){return acc>val?acc:val;});
 
-  varietyResults.forEach(function(key) {
-    table.push([key._id.key, key.value.types.toString(), key.totalOccurrences.toString(), key.percentContaining.toFixed(maxDigits).toString()]);
+  var rows = results.map(function(row) {
+    return [row._id.key, row.value.types, row.totalOccurrences, row.percentContaining.toFixed(maxDigits)];
   });
-
-  var colMaxWidth = function(arr, index) {
-   return Math.max.apply(null, arr.map(function(row){return row[index].toString().length;}));
-  };
-
+  var table = [headers, headers.map(function(){return '';})].concat(rows);
+  var colMaxWidth = function(arr, index) {return Math.max.apply(null, arr.map(function(row){return row[index].toString().length;}));};
   var pad = function(width, string, symbol) { return width <= string.length ? string : pad(width, isNaN(string) ? string + symbol : symbol + string, symbol); };
-
-  var output = '';
-  table.forEach(function(row, ri){
-    output += '| ' + row.map(function(cell, i) {return pad(colMaxWidth(table, i), cell, ri === 1 ? '-' : ' ');}).join(' | ') + ' |\n';
+  table = table.map(function(row, ri){
+    return '| ' + row.map(function(cell, i) {return pad(colMaxWidth(table, i), cell.toString(), ri === 1 ? '-' : ' ');}).join(' | ') + ' |';
   });
-  var lineLength = output.split('\n')[0].length - 2; // length of first (header) line minus two chars for edges
-  var border = '+' + pad(lineLength, '', '-') + '+';
-  print(border + '\n' + output + border);
+  var border = '+' + pad(table[0].length - 2, '', '-') + '+';
+  return [border].concat(table).concat(border).join('\n');
+};
+
+var pluginsOutput = $plugins.execute('formatResults', varietyResults);
+if (pluginsOutput.length > 0) {
+  pluginsOutput.forEach(function(i){print(i);});
+} else if(config.outputFormat === 'json') {
+  printjson(varietyResults); // valid formatted json output, compressed variant is printjsononeline()
+} else {
+   print(createAsciiTable(varietyResults)); // output nice ascii table with results
 }
 
-}()); // end strict mode
+}.bind(this)()); // end strict mode