Merge pull request #48 from todvora/master

Results printed as a ascii table (and added json switch)

test/src/main/java/com/github/variety/Variety.java

@@ -11,7 +11,9 @@ import java.io.InputStreamReader;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.StringJoiner;
 
 /**
@@ -23,12 +25,14 @@ public class Variety {
      * Hardcoded database name in variety.js for analysis results
      */
     public static final String VARIETY_RESULTS_DBNAME = "varietyResults";
+    public static final String FORMAT_JSON = "json";
+    public static final String FORMAT_ASCII = "ascii";
 
     public static final String PARAM_QUERY = "query";
     public static final String PARAM_SORT = "sort";
     public static final String PARAM_MAXDEPTH = "maxDepth";
     public static final String PARAM_LIMIT = "limit";
-
+    public static final String PARAM_OUTPUT_FORMAT = "outputFormat";
 
     private final String inputDatabase;
     private final String inputCollection;
@@ -38,8 +42,10 @@ public class Variety {
     private Integer maxDepth;
     private String query;
     private String sort;
+    private String outputFormat;
+    private boolean quiet;
 
-    private boolean verbose = true;
+    private boolean isStdoutForwarded = true;
 
     /**
      * Create variety wrapper with defined connection do analysed database and collection
@@ -99,14 +105,28 @@ public class Variety {
         return this;
     }
 
+    public Variety withFormat(final String format) {
+        this.outputFormat = format;
+        return this;
+    }
+
+    /**
+     * Wrapper for command line option '--quiet', that is passed to mongo shell. Variety is able to read this option
+     * and mute its logs with metadata.
+     */
+    public Variety withQuiet(boolean quiet) {
+        this.quiet = quiet;
+        return this;
+    }
+
     /**
      * Enable analysis output stdout of script to stdout of java process.
      * Deprecated because it should only be used for debugging of test, not real/production tests itself. If you
      * need to read stdout of variety, it can be accessed through {@link VarietyAnalysis#getStdOut()}
      */
     @Deprecated()
-    public Variety verbose() {
-        this.verbose = true;
+    public Variety withStdoutForwarded(final boolean isStdoutForwarded) {
+        this.isStdoutForwarded = isStdoutForwarded;
         return this;
     }
 
@@ -117,15 +137,26 @@ public class Variety {
      * @throws InterruptedException
      */
     public VarietyAnalysis runAnalysis() throws IOException, InterruptedException {
-        final String[] commands = new String[]{"mongo", this.inputDatabase,  "--eval", buildParams(), getVarietyPath()};
-        final Process child = Runtime.getRuntime().exec(commands);
+
+        List<String> commands = new ArrayList<>();
+        commands.add("mongo");
+        commands.add(this.inputDatabase);
+        if(quiet) {
+            commands.add("--quiet");
+        }
+        commands.add("--eval");
+        commands.add(buildParams());
+        commands.add(getVarietyPath());
+
+        final String[] cmdarray = commands.toArray(new String[commands.size()]);
+        final Process child = Runtime.getRuntime().exec(cmdarray);
 
         final int returnCode = child.waitFor();
         final String stdOut = readStream(child.getInputStream());
 
         if(returnCode != 0) {
-            throw new RuntimeException("Failed to execute variety.js with arguments: " + Arrays.toString(commands) + ".\n" + stdOut);
-        } else if(verbose) {
+            throw new RuntimeException("Failed to execute variety.js with arguments: " + Arrays.toString(cmdarray) + ".\n" + stdOut);
+        } else if(isStdoutForwarded) {
             System.out.println(stdOut);
         }
         return new VarietyAnalysis(mongoClient, inputCollection, stdOut);
@@ -154,6 +185,10 @@ public class Variety {
             args.add(PARAM_SORT + " = " + sort);
         }
 
+        if(outputFormat != null) {
+            args.add(PARAM_OUTPUT_FORMAT + " = '" + outputFormat + "'");
+        }
+
         return args.toString();
     }
 

test/src/test/java/com/github/variety/test/OutputFormatTest.java

+package com.github.variety.test;
+
+import com.github.variety.Variety;
+import com.github.variety.VarietyAnalysis;
+import com.mongodb.BasicDBList;
+import com.mongodb.util.JSON;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class OutputFormatTest {
+
+    private Variety variety;
+
+    @Before
+    public void setUp() throws Exception {
+        this.variety = new Variety("test", "users");
+        variety.getSourceCollection().insert(SampleData.getDocuments());
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        variety.getVarietyResultsDatabase().dropDatabase();
+        variety.getSourceCollection().drop();
+    }
+
+    @Test
+    public void verifyJsonEntries() throws Exception {
+        final VarietyAnalysis analysis = variety
+                .withQuiet(true) // do not output any other metadata, only results
+                .withFormat(Variety.FORMAT_JSON)
+                .runAnalysis();
+
+        // Verify, that output is parse-able json by transforming stdout to json
+        final BasicDBList parsed = (BasicDBList) JSON.parse(analysis.getStdOut());
+
+        // there should be seven different json results
+        Assert.assertEquals(7, parsed.size());
+    }
+
+    @Test
+    public void verifyAsciiTableOutput() throws Exception {
+        final VarietyAnalysis analysis = variety.withFormat(Variety.FORMAT_ASCII).runAnalysis();
+
+        // filter only lines starting with character '|'
+        final String actual = Stream.of(analysis.getStdOut().split("\n"))
+                .filter(line -> line.startsWith("|") || line.startsWith("+"))
+                .collect(Collectors.joining("\n"));
+
+        Assert.assertEquals(SampleData.EXPECTED_DATA_ASCII_TABLE, actual);
+
+    }
+}

test/src/test/java/com/github/variety/test/ParametersParsingTest.java

@@ -79,6 +79,27 @@ public class ParametersParsingTest {
         }
     }
 
+    @Test
+    public void testDefaultOutputFormatParam() throws Exception {
+        final VarietyAnalysis analysis = variety.runAnalysis(); // format option not provided
+        final Map<String, String> params = getParamsMap(analysis.getStdOut());
+        Assert.assertEquals("ascii", params.get(Variety.PARAM_OUTPUT_FORMAT));
+    }
+
+    @Test
+    public void testAsciiOutputFormatParam() throws Exception {
+        final VarietyAnalysis analysis = variety.withFormat(Variety.FORMAT_ASCII).runAnalysis();
+        final Map<String, String> params = getParamsMap(analysis.getStdOut());
+        Assert.assertEquals("ascii", params.get(Variety.PARAM_OUTPUT_FORMAT));
+    }
+
+    @Test
+    public void testJsonOutputFormatParam() throws Exception {
+        final VarietyAnalysis analysis = variety.withFormat(Variety.FORMAT_JSON).runAnalysis();
+        final Map<String, String> params = getParamsMap(analysis.getStdOut());
+        Assert.assertEquals("json", params.get(Variety.PARAM_OUTPUT_FORMAT));
+    }
+
     /**
      * @param stdout Text from mongo shell, containing variety config output + json results
      * @return Map of config values

test/src/test/java/com/github/variety/test/JsonOutputTest.java → test/src/test/java/com/github/variety/test/QuietOptionTest.java

@@ -2,19 +2,17 @@ package com.github.variety.test;
 
 import com.github.variety.Variety;
 import com.github.variety.VarietyAnalysis;
-import com.mongodb.DBObject;
-import com.mongodb.util.JSON;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.stream.Stream;
-
 /**
- * Variety outputs json-like results to stdout. Lets verify that.
+ * Variety can read '--quiet' option passed to mongo shell and mute all debug/metadata logs. In this case only
+ * results should be printed. Together with output format set to json should be possible to simply forward output
+ * from variety to another tool processing valid json.
  */
-public class JsonOutputTest {
+public class QuietOptionTest {
 
     private Variety variety;
 
@@ -30,21 +28,12 @@ public class JsonOutputTest {
         variety.getSourceCollection().drop();
     }
 
+    /**
+     * verify, that output contains only results table and nothing more
+     */
     @Test
-    public void verifyJsonEntries() throws Exception {
-        final VarietyAnalysis analysis = variety.runAnalysis();
-
-        // TODO: output itself is not valid JSON. It contains mongo shell output (can be removed with --quiet) and variety execution info.
-        // At the end of output, there are printed records from result collection, every record on new line.
-        // Valid json output is requested in issue https://github.com/variety/variety/issues/23
-
-        // Verify, that every object is parse-able json by transforming strings to json stream
-        // Results are detected by line starting with character '{'.
-        final Stream<DBObject> objects = Stream.of(analysis.getStdOut().split("\n"))
-                .filter(line -> line.startsWith("{"))
-                .map(str -> (DBObject)JSON.parse(str));
-
-        // there should be seven different json results in the stdout
-        Assert.assertEquals(7, objects.count());
+    public void testQuietLogs() throws Exception {
+        final VarietyAnalysis varietyAnalysis = variety.withQuiet(true).runAnalysis();
+        Assert.assertEquals(SampleData.EXPECTED_DATA_ASCII_TABLE, varietyAnalysis.getStdOut());
     }
 }

test/src/test/java/com/github/variety/test/SampleData.java

@@ -12,6 +12,23 @@ import java.util.List;
 class SampleData {
 
     /**
+     * Ascii table representation of sample data results. It should be possible to verify actual output of Variety
+     * against this table, to check correct formatting.
+     */
+    public static final String EXPECTED_DATA_ASCII_TABLE =
+            "+------------------------------------------------------------+\n" +
+            "| key                | types        | occurrences | percents |\n" +
+            "| ------------------ | ------------ | ----------- | -------- |\n" +
+            "| _id                | ObjectId     | 5           | 100      |\n" +
+            "| name               | String       | 5           | 100      |\n" +
+            "| bio                | String       | 3           | 60       |\n" +
+            "| pets               | String,Array | 2           | 40       |\n" +
+            "| birthday           | String       | 2           | 40       |\n" +
+            "| someBinData        | BinData-old  | 1           | 20       |\n" +
+            "| someWeirdLegacyKey | String       | 1           | 20       |\n" +
+            "+------------------------------------------------------------+";
+
+    /**
      * Java representation of sample collection provided in variety README:<p>
      *
      * {name: "Tom", bio: "A nice guy.", pets: ["monkey", "fish"], someWeirdLegacyKey: "I like Ike!"}<p>

test/src/test/java/com/github/variety/test/VersionInfoTest.java

@@ -17,7 +17,7 @@ import java.util.regex.Pattern;
  */
 public class VersionInfoTest {
 
-    public static final Pattern VARIETYJS_PATTERN = Pattern.compile("print\\('(.+), released (.+)'\\).*");
+    public static final Pattern VARIETYJS_PATTERN = Pattern.compile("\\w+\\('(.+), released (.+)'\\).*");
     public static final Pattern CHANGELOG_PATTERN = Pattern.compile("\\((.+)\\)(.+):(.*)");
     
     private List<String> varietyLines;

variety.js

@@ -8,8 +8,15 @@ finding rare keys.
 Please see https://github.com/variety/variety for details.
 
 Released by Maypop Inc, © 2012-2014, under the MIT License. */
-print('Variety: A MongoDB Schema Analyzer');
-print('Version 1.4.1, released 14 Oct 2014');
+
+var log = function(message) {
+  if(!__quiet) { // mongo shell param, coming from https://github.com/mongodb/mongo/blob/5fc306543cd3ba2637e5cb0662cc375f36868b28/src/mongo/shell/dbshell.cpp#L624
+      print(message);
+    }
+};
+
+log('Variety: A MongoDB Schema Analyzer');
+log('Version 1.4.1, released 14 Oct 2014');
 
 var dbs = [];
 var emptyDbs = [];
@@ -51,19 +58,21 @@ if (db[collection].count() === 0) {
 }
 
 if (typeof query === 'undefined') { var query = {}; }
-print('Using query of ' + tojson(query));
+log('Using query of ' + tojson(query));
 
 if (typeof limit === 'undefined') { var limit = db[collection].find(query).count(); }
-print('Using limit of ' + limit);
+log('Using limit of ' + limit);
 
 if (typeof maxDepth === 'undefined') { var maxDepth = 99; }
-print('Using maxDepth of ' + maxDepth);
+log('Using maxDepth of ' + maxDepth);
 
 if (typeof sort === 'undefined') { var sort = {_id: -1}; }
-print('Using sort of ' + tojson(sort));
+log('Using sort of ' + tojson(sort));
+
+if (typeof outputFormat === 'undefined') { var outputFormat = "ascii"; }
+log('Using outputFormat of ' + outputFormat);
 
 
-	
 varietyTypeOf = function(thing) {
   if (typeof thing === 'undefined') { throw 'varietyTypeOf() requires an argument'; }
 
@@ -160,7 +169,7 @@ db[collection].find(query).sort(sort).limit(limit).forEach(function(obj) {
 			interimResults[key]['types'][valueType] = true;
 			interimResults[key]['totalOccurrences']++;
 		}
-	} 
+	}
 });
 
 
@@ -180,15 +189,15 @@ var resultsDB = db.getMongo().getDB('varietyResults');
 var resultsCollectionName = collection + 'Keys';
 
 // replace results collection
-print('creating results collection: '+resultsCollectionName);
+log('creating results collection: '+resultsCollectionName);
 resultsDB[resultsCollectionName].drop();
 for(var result in varietyResults) {
-  resultsDB[resultsCollectionName].insert(varietyResults[result]); 
+  resultsDB[resultsCollectionName].insert(varietyResults[result]);
 }
 
 var numDocuments = db[collection].count();
 
-print('removing leaf arrays in results collection, and getting percentages');
+log('removing leaf arrays in results collection, and getting percentages');
 resultsDB[resultsCollectionName].find({}).forEach(function(key) {
   var keyName = key._id.key;
   
@@ -216,6 +225,26 @@ resultsDB[resultsCollectionName].find({}).forEach(function(key) {
 });
 
 var sortedKeys = resultsDB[resultsCollectionName].find({}).sort({totalOccurrences: -1});
-sortedKeys.forEach(function(key) {
-  print(tojson(key, '', true));
-});
+
+if(outputFormat === 'json') {
+  printjson(sortedKeys.toArray()); // valid formatted json output, compressed variant is printjsononeline()
+} else {  // output nice ascii table with results
+  var table = [["key", "types", "occurrences", "percents"], ["", "", "", ""]]; // header + delimiter rows
+  sortedKeys.forEach(function(key) {
+    table.push([key._id.key, key.value.types.toString(), key.totalOccurrences, key.percentContaining]);
+  });
+
+  var colMaxWidth = function(arr, index) {
+   return Math.max.apply(null, arr.map(function(row){return row[index].toString().length;}));
+  };
+
+  var pad = function(width, string, symbol) { return (width <= string.length) ? string : pad(width, string + symbol, symbol); };
+
+  var output = "";
+  table.forEach(function(row, ri){
+    output += ("| " + row.map(function(cell, i) {return pad(colMaxWidth(table, i), cell, ri == 1 ? "-" : " ");}).join(" | ") + " |\n");
+  });
+  var lineLength = output.split("\n")[0].length - 2; // length of first (header) line minus two chars for edges
+  var border = "+" + pad(lineLength, "", "-") + "+";
+  print(border + "\n" + output + border);
+}