Merge pull request #111 from todvora/master

Tests rewritten to JavaScript, npm package definition, ESLint

.babelrc

+{
+ "presets": ["es2015" ]
+}

.eslintrc.js

+module.exports = {
+    "env": {
+        "mongo": true,
+        "node": true,
+        "es6": true,
+        "mocha": true
+    },
+    "extends": "eslint:recommended",
+    "rules": {
+      "indent": [
+          2,
+          2
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "single"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "brace-style": [
+          2,
+          "1tbs",
+          { "allowSingleLine": true }
+        ]
+    },
+    "globals": {
+        "__quiet": false,
+        "slaveOk": false,
+        "collection": false,
+        "DBQuery": false,
+        "BinData": false,
+        "tojson": false
+
+    },
+    "parserOptions": {
+      "sourceType": "module"
+    }
+};

.gitignore

 *~
-
+node_modules/
 # intellij idea project files
 *.iml
 *.ipr
-.idea/
+.idea/
\ No newline at end of file

.jshintrc

-{
-  "bitwise": true,
-  "curly": true,
-  "eqeqeq": true,
-  "freeze": true,
-  "globals": {
-    "__quiet": false,
-    "collection": false,
-    "print": false,
-    "db": false,
-    "tojson": false,
-    "ObjectId": false,
-    "BinData": false,
-    "DBQuery": false,
-    "printjson": false,
-    "load": false,
-    "module": false,
-
-    "query": true,
-    "limit": true,
-    "maxDepth": true,
-    "sort": true,
-    "outputFormat": true,
-    "persistResults": true,
-    "plugins": true,
-    "slaveOk" : true,
-    "connect": true
-  },
-  "latedef": true,
-  "singleGroups": true,
-  "strict": true,
-  "undef": true,
-  "unused": true
-}

.travis.yml

 language: node_js
 node_js:
-  - "4.1"
+  - '5.0'
 sudo: required
 services: docker
-install:
- - npm install jshint -g
 env:
  matrix:
    - MONGODB_VERSION=2.4
@@ -13,5 +11,4 @@ env:
    - MONGODB_VERSION=3.0
    - MONGODB_VERSION=3.2
 script:
- - jshint variety.js
- - ./test.sh
+ - npm run travis-ci
\ No newline at end of file

Dockerfile.template

-FROM mongo:{MONGODB_VERSION}
-
-## Java installation
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
-RUN echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' | tee /etc/apt/sources.list.d/webupd8team-java-trusty.list
-RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
-RUN apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends oracle-java8-installer && apt-get clean all
-ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
-RUN java -version
-
-## Maven installation
-RUN apt-get install -y --force-yes --no-install-recommends curl
-RUN apt-get install -y --force-yes --no-install-recommends ca-certificates
-ENV MAVEN_VERSION 3.3.9
-RUN apt-get install -y --force-yes --no-install-recommends curl
-RUN curl -fsSL https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - -C /usr/share \
-  && mv /usr/share/apache-maven-$MAVEN_VERSION /usr/share/maven \
-  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
-ENV MAVEN_HOME /usr/share/maven
-RUN mvn --version
-
-## Prepare entrypoint script (start MongoDB, switch to test dir, execute maven)
-RUN mkdir /opt/variety-test
-WORKDIR /opt/variety-test/
-RUN echo "#!/bin/sh" > run.sh
-RUN echo "mongod --logpath /var/log/mongod.log &" >> run.sh
-RUN echo "cd /opt/variety/test" >> run.sh
-RUN echo "mvn test" >> run.sh
-RUN chmod +x /opt/variety-test/run.sh
-
-ENTRYPOINT ["/opt/variety-test/run.sh"]
\ No newline at end of file

README.markdown

@@ -174,7 +174,7 @@ $ mongo test --quiet --eval "var collection = 'users', persistResults=true, resu
 
 ### Command Line Interface
 Variety itself is command line friendly, as shown on examples above.
-But if you are a NPM and Node.js user, you could prefer the 
+But if you are a NPM and Node.js user, you could prefer the
 [variety-cli](https://github.com/variety/variety-cli) project. It simplifies usage of
 Variety and removes all the complexity of passing variables in the ```--eval``` argument and
 providing a path to the variety.js library.
@@ -195,6 +195,29 @@ A Mongo collection does not enforce a predefined schema like a relational databa
 
 Absolutely none, except MongoDB. Written in 100% JavaScript. _(mongod's "noscripting" may not be set to true, and 'strict mode' must be disabled.)_
 
+##### Development, Hacking #####
+This project is NPM based and provides standard NPM functionality. As an additional (not required) dependency, [Docker](https://www.docker.com/) can be installed to test against different MongoDB versions.
+
+To install all dev dependencies call as usual:
+```
+npm install
+```
+
+By default, tests expect MongoDB available on ```localhost:27017``` and can be executed by calling:
+
+```
+npm test
+```
+
+If you have Docker installed and don't want to test against your own MongoDB instance,
+you can execute tests against dockerized MongoDB:
+
+```
+MONGODB_VERSION=3.2 npm run test:docker
+```
+The script downloads one of [official MongoDB images](https://hub.docker.com/_/mongo/) (based on your provided version),
+starts the database, executes test suite against it (inside the container) and stops the DB.
+
 #### Reporting Issues / Contributing ####
 
 Please report any bugs and feature requests on the Github issue tracker. I will read all reports!
@@ -203,15 +226,15 @@ I accept pull requests from forks. Very grateful to accept contributions from fo
 
 #### Core Maintainers ####
 
-* Tomáš Dvořák ([personal website] (http://www.tomas-dvorak.cz/))
-* Eve Freeman ([Twitter] (https://twitter.com/wefreema))
-* James Cropcho (original creator of Variety) ([Twitter] (https://twitter.com/Cropcho))
+* Tomáš Dvořák ([personal website](http://www.tomas-dvorak.cz/))
+* Eve Freeman ([Twitter](https://twitter.com/wefreema))
+* James Cropcho (original creator of Variety) ([Twitter](https://twitter.com/Cropcho))
 
 #### Special Thanks ####
 
-Additional special thanks to Gaëtan Voyer-Perraul ([@gatesvp] (https://twitter.com/#!/@gatesvp)) and Kristina Chodorow ([@kchodorow] (https://twitter.com/#!/kchodorow)) for answering other people's questions about how to do this on Stack Overflow, thereby providing me with the initial seed of code which grew into this tool.
+Additional special thanks to Gaëtan Voyer-Perraul ([@gatesvp](https://twitter.com/#!/@gatesvp)) and Kristina Chodorow ([@kchodorow](https://twitter.com/#!/kchodorow)) for answering other people's questions about how to do this on Stack Overflow, thereby providing me with the initial seed of code which grew into this tool.
 
-Much thanks also, to Kyle Banker ([@Hwaet] (https://twitter.com/#!/hwaet)) for writing an unusually good book on MongoDB, which has taught me everything I know about it so far.
+Much thanks also, to Kyle Banker ([@Hwaet](https://twitter.com/#!/hwaet)) for writing an unusually good book on MongoDB, which has taught me everything I know about it so far.
 
 #### Tools Which Use Variety (Open Source) ####
 

docker/Dockerfile.template

+FROM mongo:{MONGODB_VERSION}
+
+RUN apt-get -qq update
+RUN apt-get install -y --force-yes --no-install-recommends curl
+
+# This is the recommended installation of node
+# See: https://nodejs.org/en/download/package-manager/#debian-and-ubuntu-based-linux-distributions
+# RUN curl -sL https://deb.nodesource.com/setup_5.x | bash -
+# RUN apt-get install -y --force-yes --no-install-recommends nodejs
+
+# To speed up the installation, we skip packages and download directly node archive
+# Version of node is determinded by Heroku's API https://semver.io/node/stable
+RUN NODE_VERSION=$(curl -sk https://semver.io/node/stable) \
+  && curl -SLO "http://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
+  && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.gz"
+
+ENTRYPOINT ["/opt/variety/docker/start-script.sh"]

docker/start-script.sh

+#!/bin/sh
+
+# Start script for testing container
+
+# start MongoDB with disabled logs and journal
+mongod --nojournal --logpath /dev/null &
+
+# switch to linked sources volume
+cd /opt/variety
+
+# install all dependencies
+npm install
+
+# Wait until the DB is started and responds on selected port
+while ! curl --silent http://localhost:27017  > /dev/null 2>&1
+do
+  echo "waiting for MongoDB connection"
+  sleep 1
+done
+
+echo "MongoDB ready on port 27017"
+
+# start actual tests
+npm test

package.json

+{
+  "name": "variety",
+  "version": "1.5.0",
+  "description": "A schema analyzer for MongoDB",
+  "main": "variety.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "lint": "node_modules/.bin/eslint variety.js spec",
+    "lint:fix": "node_modules/.bin/eslint variety.js spec --fix",
+    "test": "node_modules/.bin/mocha --compilers js:babel-core/register --recursive --reporter spec --timeout 15000 spec",
+    "test:docker": "./test.sh",
+    "travis-ci": "npm run lint && npm run test:docker"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/variety/variety.git"
+  },
+  "author": "James Cropcho (https://twitter.com/Cropcho)",
+  "contributors": [
+    "James Cropcho (https://twitter.com/Cropcho)",
+    "Eve Freeman (https://twitter.com/wefreema)",
+    "Tomas Dvorak <todvora@gmail.com> (http://www.tomas-dvorak.cz/)"
+  ],
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/variety/variety/issues"
+  },
+  "homepage": "https://github.com/variety/variety#readme",
+  "devDependencies": {
+    "babel-core": "^6.7.2",
+    "babel-preset-es2015": "^6.6.0",
+    "child-process-promise": "^1.1.0",
+    "eslint": "^2.4.0",
+    "mocha": "^2.4.5",
+    "mongodb": "^2.1.7",
+    "q": "^1.4.1"
+  }
+}

spec/BasicAnalysisTest.js

+const assert = require('assert');
+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+const sampleData = require('./assets/SampleData');
+const expectedAscii = require('./assets/ExpectedAscii');
+
+describe('Basic Analysis', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should return ASCII results', () => {
+    return test.runAnalysis({collection:'users'}, true).then(output => {
+      assert.equal(output, expectedAscii);
+    });
+  });
+
+  it('should return JSON results', () => {
+    return test.runJsonAnalysis({collection:'users'}, true)
+      .then(results => {
+
+        results.validateResultsCount(7);
+
+        results.validate('_id', 5, 100.0, {ObjectId: 5});
+        results.validate('name', 5, 100.0, {String: 5});
+        results.validate('bio', 3, 60.0, {String: 3});
+        results.validate('birthday', 2,  40.0, {Date: 2});
+        results.validate('pets', 2,  40.0, {String: 1, Array: 1});
+        results.validate('someBinData', 1,  20.0, {'BinData-generic': 1});
+        results.validate('someWeirdLegacyKey', 1,  20.0, {String: 1});
+      });
+  });
+});

spec/DatatypeRecognitionTest.js

+const Binary = require('mongodb').Binary;
+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+const crazyObject = {
+  key_string: 'Just plain String',
+  key_boolean: true,
+  key_number: 1,
+  key_date: new Date(),
+  'key_binData-generic': new Binary('1234'), // TODO: how to create other bin-data types?
+  key_array: [],
+  key_object: {},
+  key_null: null
+};
+
+describe('Data type recognition', () => {
+
+  beforeEach(() => test.init([crazyObject]));
+  afterEach(() => test.cleanUp());
+
+  it('should recognize all supported data types', () => {
+    return test.runJsonAnalysis({collection:'users'}, true)
+      .then(results => {
+        results.validate('_id', 1, 100.0, {ObjectId: 1});
+        results.validate('key_string', 1, 100.0, {String: 1});
+        results.validate('key_number', 1, 100.0, {Number: 1});
+        results.validate('key_date', 1, 100.0, {Date: 1});
+        results.validate('key_binData-generic', 1, 100.0, {'BinData-generic': 1});
+        results.validate('key_array', 1, 100.0, {Array: 1});
+        results.validate('key_object', 1, 100.0, {Object: 1});
+        results.validate('key_null', 1, 100.0, {null: 1}); // TODO: why has 'null' first letter lowercase, unlike all other types?
+      });
+  });
+});

spec/LimitResultsAnalysisTest.js

+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+const sampleData = require('./assets/SampleData');
+
+describe('Limited results count analysis', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should analyze only first item', () => {
+    // limit=1 without other params selects the last inserted document (see sampleData)
+    // it should equals {name: "Jim", someBinData: new BinData(2,"1234")}
+    return test.runJsonAnalysis({collection:'users', limit:1}).then(results => {
+      results.validate('_id', 1, 100.0, {ObjectId:1});
+      results.validate('name', 1, 100.0, {String:1});
+      results.validate('someBinData', 1, 100.0, {'BinData-generic':1});
+    });
+  });
+
+  it('should analyze all and compute real percentages', () => {
+    return test.runJsonAnalysis({collection:'users', limit:10}).then(results => {
+      // limit is set to higher number, that the actual number of documents in collection
+      // analysis should compute percentages based on the real number of documents, not on the
+      // number provided in the limit var.
+      results.validateResultsCount(7);
+      results.validate('_id', 5, 100.0, {ObjectId: 5});
+      results.validate('name', 5, 100.0, {String: 5});
+      results.validate('bio', 3, 60.0, {String: 3});
+      results.validate('birthday', 2,  40.0, {Date: 2});
+      results.validate('pets', 2,  40.0, {String: 1, Array: 1});
+      results.validate('someBinData', 1,  20.0, {'BinData-generic': 1});
+      results.validate('someWeirdLegacyKey', 1,  20.0, {String: 1});
+    });
+  });
+});

spec/LimitedAccessTest.js

+describe('Limited access test', () => {
+  it('TODO: should handle authenticated and unknown users');
+  // How to implement? Start another db with auth and configure users?
+});

spec/MaxDepthAnalysisTest.js

+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+describe('Max-depth-limited analysis', () => {
+
+  beforeEach(() => test.init([{name:'Walter', someNestedObject:{a:{b:{c:{d:{e:1}}}}}}]));
+  afterEach(() => test.cleanUp());
+
+  it('should return all keys', () => {
+    return test.runJsonAnalysis({collection:'users'}).then(results => {
+
+      results.validateResultsCount(8);
+
+      results.validate('_id', 1, 100.0, {ObjectId:1});
+      results.validate('name', 1, 100.0, {String:1});
+      results.validate('someNestedObject', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a.b', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a.b.c', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a.b.c.d', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a.b.c.d.e', 1, 100.0, {Number:1});
+    });
+  });
+
+  it('should return only first 3 levels', () => {
+    return test.runJsonAnalysis({collection:'users', maxDepth:3}).then(results => {
+
+      results.validateResultsCount(5);
+
+      results.validate('_id', 1, 100.0, {ObjectId:1});
+      results.validate('name', 1, 100.0, {String:1});
+      results.validate('someNestedObject', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a', 1, 100.0, {Object:1});
+      results.validate('someNestedObject.a.b', 1, 100.0, {Object:1});
+    });
+  });
+});

spec/ParametersParsingTest.js

+const assert = require('assert');
+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+const sampleData = require('./assets/SampleData');
+
+const parseParams = (output) => {
+  return output
+    .split('\n') // split by new line
+    .filter(line => line.indexOf('Using') === 0) // take only lines starting with Using
+    .map(line => /^Using\s{1}(\w+)\s{1}of\s{1}(.*)$/.exec(line)) // parse with regular expression
+    .reduce((acc, match) => {acc[match[1]] = JSON.parse(match[2]); return acc;}, {}); // reduce to params object
+};
+
+describe('Parameters parsing', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should parse default params', () => {
+    return test.runAnalysis({collection:'users'})
+      .then(parseParams)
+      .then(params => {
+        assert.equal(params.collection, 'users');
+        assert.deepEqual(params.query, {});
+        assert.equal(params.limit, 5);
+        assert.equal(params.maxDepth, 99);
+        assert.deepEqual(params.sort, {'_id':-1});
+        assert.equal(params.outputFormat, 'ascii');
+        assert.equal(params.persistResults, false);
+        assert.equal(params.resultsDatabase, 'varietyResults');
+        assert.equal(params.resultsCollection, 'usersKeys');
+        assert.equal(params.resultsUser, null);
+        assert.equal(params.resultsPass, null);
+        assert.deepEqual(params.plugins, []);
+      });
+  });
+
+  it('should parse restricted results', () => {
+
+    const criteria = {
+      collection:'users',
+      query: {name:'Harry'},
+      sort: {name:1},
+      maxDepth: 5,
+      limit: 2
+    };
+
+    return test.runAnalysis(criteria)
+      .then(parseParams)
+      .then(params => {
+        assert.equal(params.limit, 2);
+        assert.equal(params.maxDepth, 5);
+        assert.deepEqual(params.sort, {name:1});
+        assert.deepEqual(params.query, {name:'Harry'});
+      });
+  });
+
+  it('should recognize unknown collection', () => {
+    return test.runAnalysis({collection:'--unknown--'})
+      .then(() => {throw new Error('Should fail and be handled in catch branch');})
+      .catch(err => {
+        assert.ok(err.code > 0);
+        assert.ok(err.stdout.indexOf('The collection specified (--unknown--) in the database specified (test) does not exist or is empty.') > -1);
+      });
+  });
+
+});

spec/PersistenceTest.js

+const assert = require('assert');
+const Tester = require('./utils/Tester.js');
+
+const test = new Tester('test', 'users');
+
+const sampleData = require('./assets/SampleData');
+
+describe('Persistence of results', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should persist results into varietyResults DB', () => {
+    return test.runAnalysis({collection:'users', persistResults: true}, true)
+      .then(() => test.getDb('varietyResults'))
+      .then((db) => db.collection('usersKeys').find().toArray())
+      .then((arr) => {
+        assert.equal(arr.length, 7);
+        let keys = arr.map(it => it._id.key);
+        assert.deepEqual(keys, ['_id', 'name', 'bio', 'birthday', 'pets', 'someBinData', 'someWeirdLegacyKey']);
+      });
+  });
+});

spec/PluginTest.js

+const assert = require('assert');
+const Tester = require('./utils/Tester.js');
+const path = require('path');
+const test = new Tester('test', 'users');
+
+const sampleData = require('./assets/SampleData');
+
+const expectedOutput = `
+key|types|occurrences|percents
+_id|ObjectId|5|100
+name|String|5|100
+bio|String|3|60
+birthday|Date|2|40
+pets|Array,String|2|40
+someBinData|BinData-generic|1|20
+someWeirdLegacyKey|String|1|20
+`.trim();
+
+const getPluginPath = () => path.resolve(path.join(__dirname , 'assets', 'csvplugin.js'));
+
+describe('Plugins', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should load plugin and modify output', () => {
+    return test.runAnalysis({collection:'users', plugins: getPluginPath()}, true).then(output => {
+      assert.equal(output, expectedOutput);
+    });
+  });
+
+  it('should read additional plugin params', () => {
+    return test.runAnalysis({collection:'users', plugins: getPluginPath() + '|delimiter=;'}, true).then(output => {
+      const expectedWithSeparator = expectedOutput.replace(/\|/g, ';');
+      assert.equal(output, expectedWithSeparator);
+    });
+  });
+
+});

spec/QueryLimitedAnalysisTest.js

+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+const sampleData = require('./assets/SampleData');
+
+describe('Query-limited analysis', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should return only filtered values', () => {
+    return test.runJsonAnalysis({collection:'users', query:{birthday:{$exists: true}}}).then(results => {
+      results.validateResultsCount(5);
+      results.validate('_id', 2, 100.0, {ObjectId: 2});
+      results.validate('birthday', 2, 100.0, {Date: 2});
+      results.validate('name', 2, 100.0, {String: 2});
+      results.validate('bio', 1, 50.0, {String: 1});
+      results.validate('pets', 1, 50.0, {String: 1});
+    });
+  });
+});

spec/SortedAnalysisTest.js

+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+const sampleData = require('./assets/SampleData');
+
+describe('Sorted-data analysis', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should not exclude any results', () => {
+    return test.runJsonAnalysis({collection:'users', sort:{name:-1}}).then(results => {
+      results.validateResultsCount(7);
+      results.validate('_id', 5, 100.0, {ObjectId: 5});
+      results.validate('name', 5, 100.0, {String: 5});
+      results.validate('bio', 3, 60.0, {String: 3});
+      results.validate('birthday', 2,  40.0, {Date: 2});
+      results.validate('pets', 2,  40.0, {String: 1, Array: 1});
+      results.validate('someBinData', 1,  20.0, {'BinData-generic': 1});
+      results.validate('someWeirdLegacyKey', 1,  20.0, {String: 1});
+    });
+  });
+
+  it('should sort and apply limit', () => {
+    const criteria = {
+      collection:'users',
+      sort:{name:-1},
+      limit:1
+    };
+
+    // when sorting default SampleData by name desc, first entry becomes Tom. He is only with key 'someWeirdLegacyKey'
+    // Together with applying limit 1, Tom is the only result in analysis. That gives us chance to assume keys and verify
+    // that ordering is correct.
+    // {name: "Tom", bio: "A nice guy.", pets: ["monkey", "fish"], someWeirdLegacyKey: "I like Ike!"}
+    return test.runJsonAnalysis(criteria).then(results => {
+      results.validateResultsCount(5);
+      results.validate('_id', 1, 100.0, {ObjectId: 1});
+      results.validate('name', 1, 100.0, {String: 1});
+      results.validate('bio', 1, 100.0, {String: 1});
+      results.validate('pets', 1, 100.0, {Array: 1});
+      results.validate('someWeirdLegacyKey', 1, 100.0, {String: 1});
+    });
+  });
+});

spec/UnnamedObjectsAnalysisTest.js

+const Tester = require('./utils/Tester.js');
+const test = new Tester('test', 'users');
+
+const sampleData = [
+  {title:'Article 1', comments:[{author:'John', body:'it works', visible:true }]},
+  {title:'Article 2', comments:[{author:'Tom', body:'thanks'}, {author:'Mark', body:1}]}
+];
+
+// Test, how variety handles objects, that are not named (for example objects inside array).
+// It addresses behavior described in issue https://github.com/variety/variety/issues/29
+
+describe('Unnamed object analysis', () => {
+
+  beforeEach(() => test.init(sampleData));
+  afterEach(() => test.cleanUp());
+
+  it('should handle keys of unnamed object', () => {
+    return test.runJsonAnalysis({collection:'users'}, true)
+      .then(results => {
+        results.validateResultsCount(6);
+        results.validate('_id', 2, 100.0, {ObjectId: 2});
+        results.validate('title', 2, 100.0, {String: 2});
+        results.validate('comments', 2, 100.0, {Array: 2});
+
+        // unnamed objects are prefixed with .XX key
+        results.validate('comments.XX.author', 2, 100.0, {String: 2});
+        results.validate('comments.XX.body', 2, 100.0, {String: 2, Number:1});
+        results.validate('comments.XX.visible', 1, 50.0, {Boolean: 1});
+      });
+  });
+});

test/src/test/resources/expected_ascii_table.txt → spec/assets/ExpectedAscii.js

+module.exports =`
+
 +--------------------------------------------------------------------+
 | key                | types                | occurrences | percents |
 | ------------------ | -------------------- | ----------- | -------- |
 | _id                | ObjectId             |           5 |    100.0 |
 | name               | String               |           5 |    100.0 |
 | bio                | String               |           3 |     60.0 |
-| birthday           | String               |           2 |     40.0 |
+| birthday           | Date                 |           2 |     40.0 |
 | pets               | String (1),Array (1) |           2 |     40.0 |
-| someBinData        | BinData-old          |           1 |     20.0 |
+| someBinData        | BinData-generic      |           1 |     20.0 |
 | someWeirdLegacyKey | String               |           1 |     20.0 |
-+--------------------------------------------------------------------+
++--------------------------------------------------------------------+
\ No newline at end of file
+
+`.trim();

spec/assets/SampleData.js

+var Binary = require('mongodb').Binary;
+
+module.exports = [{
+  'name': 'Tom',
+  'bio': 'A nice guy.',
+  'pets': ['monkey', 'fish'],
+  'someWeirdLegacyKey': 'I like Ike!'
+}, {
+  'name': 'Dick',
+  'bio': 'I swordfight.',
+  'birthday': new Date(1974,2,14)
+}, {
+  'name': 'Harry',
+  'pets': 'egret',
+  'birthday': new Date(1984,2,14)
+}, {
+  'name': 'Geneviève',
+  'bio': 'Ça va?'
+}, {
+  'name': 'Jim',
+  'someBinData': new Binary('1234')  //Binary.SUBTYPE_BYTE_ARRAY
+}];

spec/utils/JsonValidator.js

+'use strict';
+
+const assert = require('assert');
+
+class JsonValidator {
+  constructor(results) {
+    this.results = results;
+  }
+
+  validate(key, totalOccurrences, percentContaining, types) {
+    let row = this.results.filter(item => item._id.key === key)[0];
+    if(typeof row === 'undefined') {
+      throw new Error(`Key '${key}' is not present in results`);
+    }
+    assert.equal(row.totalOccurrences, totalOccurrences, `TotalOccurrences of key ${key} does not match`);
+    assert.equal(row.percentContaining, percentContaining, `PercentContaining of key ${key} does not match`);
+    assert.deepEqual(row.value.types, types, `Types of key ${key} do not match`);
+  }
+
+  validateResultsCount(count) {
+    assert.equal(this.results.length, count, 'Total count of results does not match expected count.');
+  }
+}
+
+module.exports = JsonValidator;

spec/utils/MongoShell.js

+'use strict';
+
+const exec = require('child-process-promise').exec;
+
+const execute = (database, credentials, args, script, quiet, port) => {
+  let commands = ['mongo'];
+
+  commands.push('--port');
+  commands.push(port);
+
+  if (database) {
+    commands.push(database);
+  }
+  if (quiet) {
+    commands.push('--quiet');
+  }
+
+  if (credentials) {
+    commands.push('--username');
+    commands.push(credentials.username);
+    commands.push('--password');
+    commands.push(credentials.password);
+    commands.push('--authenticationDatabase');
+    commands.push(credentials.authDatabase);
+  }
+
+  if (args) {
+    commands.push('--eval');
+    commands.push(args);
+  }
+
+  if (script) {
+    commands.push(script);
+  }
+
+  return exec(commands.join(' '))
+    .then(result => result.stdout.trim());
+};
+
+module.exports = {
+  execute:execute
+};

spec/utils/Tester.js

+'use strict';
+
+const path = require('path');
+const Q = require('q');
+const MongoClient = require('mongodb').MongoClient;
+const shell = require('./MongoShell');
+const JsonValidator = require('./JsonValidator');
+
+const mongodb_port = process.env.MONGODB_PORT || 27017;
+const default_url = `mongodb://localhost:${mongodb_port}/test?autoReconnect=true`;
+
+class Tester {
+  constructor(databaseName, collectionName) {
+    this.databaseName = databaseName;
+    this.collectionName = collectionName;
+  }
+
+  connect() {
+    return Q.nfcall(MongoClient.connect, default_url)
+      .then((connection) => {
+        this.connection = connection;
+        this.coll = connection.db(this.databaseName).collection(this.collectionName);
+        return connection;
+      });
+  }
+
+  init(initialData) {
+    return this.connect()
+      .then(() => this.coll.deleteMany())
+      .then(() => this.coll.insertMany(initialData))
+      .then(() => this.connection);
+  }
+
+  cleanUp() {
+    return this.coll.deleteMany()
+      .then(() => this.connection.close());
+  }
+
+  getDb(dbName) {
+    return Q(this.connection.db(dbName));
+  }
+
+  getVarietyPath() {
+    return path.resolve(path.join(__dirname , '..', '..', 'variety.js'));
+  }
+
+  runJsonAnalysis(options) {
+    options.outputFormat = 'json';
+    return this.runAnalysis(options, true)
+      .then(JSON.parse)
+      .then(data => new JsonValidator(data));
+  }
+
+
+  runAnalysis(options, quiet) {
+    let str = [];
+    if(options) {
+      for(let key in options) {
+        let value = JSON.stringify(options[key]).replace(/"/g, '\'').replace(/\$/g, '\\$');
+        str.push(`var ${key}=${value}`);
+      }
+    }
+    return shell.execute(this.database, null, '"' + str.join(';') + '"', this.getVarietyPath(), quiet, mongodb_port);
+  }
+}
+
+module.exports = Tester;

test.sh

@@ -4,20 +4,27 @@ set -e
 # location of this script
 DIR=$(readlink -f $(dirname $0))
 
+# Read version info from env property MONGODB_VERSION or use 2.6 as default
 VERSION=${MONGODB_VERSION:=2.6}
 
+# Read Variety.js version from package.json
+PACKAGE_VERSION=$(node -p -e "require('./package.json').version")
+
 echo
 echo "****************************************"
 echo "* "
-echo "* Testing Variety.js with MongoDB $VERSION"
+echo "* Variety.js version $PACKAGE_VERSION"
+echo "* MongoDB version $VERSION"
 echo "* $(docker --version)"
 echo "* "
 echo "****************************************"
 echo
 
-sed -e "s/{MONGODB_VERSION}/$VERSION/g" Dockerfile.template > Dockerfile_$VERSION
+sed -e "s/{MONGODB_VERSION}/$VERSION/g" docker/Dockerfile.template > Dockerfile_$VERSION
+
+echo "Building docker image for Variety tests..."
 
-docker build -t variety-$VERSION -f Dockerfile_$VERSION .
+docker build -t variety-$VERSION -f Dockerfile_$VERSION . 
 docker run -t -v $DIR:/opt/variety variety-$VERSION
 
-rm Dockerfile_$VERSION
+rm Dockerfile_$VERSION
\ No newline at end of file

test/README.markdown

-## Variety tests
-Tests are primary configured for [Travis-CI](https://travis-ci.org/variety/variety) platform. See `.travis.yml` in repository (`script` section).
-
-## Dependencies
-[MongoDB](http://www.mongodb.org) installed, of course. Tests are written in [JUnit](http://junit.org/), using [Java 8](http://http://docs.oracle.com/javase/8/). [Maven 3](https://maven.apache.org/) is required.
-You should have Java 8 and Maven installed. Junit and other dependencies are then automatically handled by Maven (see `test/pom.xml`).
-
-## Run tests locally
-
-Assuming running MongoDB, go to directory `variety/test` (you should see `pom.xml` there) and run `mvn test`.
-
-Main indicator of tests result is [exit code](http://tldp.org/LDP/abs/html/exit-status.html) of script.
-In case of everything went well, return code is `0`. In case of tests fail, exit code is set to nonzero. Exit code is monitored by Travis-CI and informs about tests success or fail.
-Tests produce verbose log messages for detecting problems and errors.
-
-## Java wrapper
-The bridge between JUnit tests and Variety written in pure JS is [Java wrapper for variety](https://github.com/variety/variety/blob/master/test/src/main/java/com/github/variety/Variety.java).
-Currently the wrapper is optimized for tests usage and allows:
-
-- Specify analyzed DB and collection name
-- Forward all config parameters to Variety.js
-- execute mongo shell with all the config values and path to Variety
-- Collect standard output of mongo shell
-- Verify results values (assertion)
-- Access mongo database through native java driver (initialization, cleanup)
-
-Wrapper can be created with this command:
-
-```
-Variety wrapper = new Variety("test", "users");
-```
-
-where the first parameter is analyzed database name and second analyzed collection name. Wrapper is written following
-[builder pattern](https://en.wikipedia.org/wiki/Builder_pattern):
-
-```
-ResultsValidator analysis = new Variety("test", "users")
-                .withMaxDepth(10)
-                .withSort("{name:-1}")
-                .withLimit(5)
-                .runDatabaseAnalysis();
-```
-
-```ResultsValidator``` is the actual analysis result. Main purpose is to easy verify results:
-
-```
-validate(String key, double totalOccurrences, double percentContaining, String... types)
-```
-If the result does not match expectations, AssertionError is thrown (standard JUnit behavior). There are two possibilities,
-how to obtain results. Variety can store results in collection in MongoDB, or output results as a valid JSON to standard
-output. This two ways have own representations in wrapper:
-
-- runDatabaseAnalysis
-- runJsonAnalysis
-
-Both of them preset important options for Variety (quiet, persistResults, outputFormat) to comply with validator.
-
-
-## Tests lifecycle
- - Initialization, prepare data. Every test has method annotated with `@Before`.
- - Variety analysis, run variety.js against prepared data and verify results. See `Variety.java`, method `runDatabaseAnalysis()` and methods annotated with `@Test`.
- - Resources cleanup, see method annotated with `@After`.
-
-## Used databases and collections
-Tests use two databases, `test` and `varietyResults`. In DB `test`, there will be created collection `users`.
-Collection is later analyzed by variety and results stored in DB `varietyResults`, collection `usersKeys`.
-
-Cleanup method should remove both test and analysis data. In case of JSON validator, there is no results db/collection created.
-
-## Contribute
-You can extend current test cases or create new JUnit test. All tests under `test/src/test/` are automatically included into run.
\ No newline at end of file

test/pom.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>com.github.variety</groupId>
-    <artifactId>test</artifactId>
-    <version>1.0-SNAPSHOT</version>
-
-    <properties>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.mongodb</groupId>
-            <artifactId>mongo-java-driver</artifactId>
-            <version>3.2.1</version>
-        </dependency>
-
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.11</version>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.1</version>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-</project>
\ No newline at end of file

test/src/main/java/com/github/variety/Credentials.java

-package com.github.variety;
-
-import com.mongodb.BasicDBObjectBuilder;
-import com.mongodb.DBObject;
-import com.mongodb.MongoCredential;
-import com.mongodb.util.JSON;
-
-/**
- * MongoDB access credentials for admin/root with unrestricted access and for user with read only database test
- */
-public enum Credentials {
-    ADMIN("admin", "variety_test_admin", "admin", "['userAdminAnyDatabase', 'readWriteAnyDatabase', 'dbAdminAnyDatabase']"),
-    USER("test", "variety_test_user", "test", "['read']");
-
-    /**
-     * Name of database, where auth objects are stored.
-     */
-    public static final String AUTH_DATABASE_NAME = "admin";
-
-    private final String authDatabase;
-    private final String username;
-    private final String password;
-    private final String rolesJson;
-
-    Credentials(final String authDatabase, final String username, final String password, final String rolesJson) {
-        this.authDatabase = authDatabase;
-        this.username = username;
-        this.password = password;
-        this.rolesJson = rolesJson;
-    }
-
-    public String getAuthDatabase() {
-        return authDatabase;
-    }
-
-    public String getUsername() {
-        return username;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    /**
-     * @return Auth credentials for MongoDB Java driver.
-     */
-    public MongoCredential getMongoCredential() {
-        return MongoCredential.createMongoCRCredential(getUsername(), getAuthDatabase(), getPassword().toCharArray());
-    }
-
-    /**
-     * Convert username, password and roles to MongoDB document for user creation
-     * @return json document to be passed to createUser (mongodb version >=2.6.x) / addUser function(mongodb 2.4.x)
-     */
-    public DBObject getUserDocument() {
-        return new BasicDBObjectBuilder()
-                .add("user", username)
-                .add("pwd", password)
-                .add("roles", JSON.parse(this.rolesJson))
-                .get();
-    }
-}

test/src/main/java/com/github/variety/MongoShell.java

-package com.github.variety;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.StringJoiner;
-
-public class MongoShell {
-
-    private final boolean quiet;
-    private final Credentials credentials;
-    private final String eval;
-    private final String database;
-    private final String script;
-
-    public MongoShell(final String database, final Credentials credentials, final String eval, final String script, final boolean quiet) {
-        this.quiet = quiet;
-        this.credentials = credentials;
-        this.eval = eval;
-        this.database = database;
-        this.script = script;
-    }
-
-    public String execute() throws IOException, InterruptedException {
-        final List<String> commands = new ArrayList<>();
-        commands.add("mongo");
-        if (database != null && !database.isEmpty()) {
-            commands.add(this.database);
-        }
-        if (quiet) {
-            commands.add("--quiet");
-        }
-
-        if (credentials != null) {
-            commands.add("--username");
-            commands.add(credentials.getUsername());
-            commands.add("--password");
-            commands.add(credentials.getPassword());
-            commands.add("--authenticationDatabase");
-            commands.add(credentials.getAuthDatabase());
-        }
-
-        if (eval != null && !eval.isEmpty()) {
-            commands.add("--eval");
-            commands.add(eval);
-        }
-
-        if (script != null && !script.isEmpty()) {
-            commands.add(script);
-        }
-
-        final String[] cmdarray = commands.toArray(new String[commands.size()]);
-        final Process child = Runtime.getRuntime().exec(cmdarray);
-
-        final int returnCode = child.waitFor();
-        final String stdOut = readStream(child.getInputStream());
-
-        if (returnCode != 0) {
-            throw new RuntimeException("Failed to execute MongoDB shell with arguments: " + Arrays.toString(cmdarray) + ".\n" + stdOut);
-        }
-        return stdOut;
-    }
-
-    /**
-     * Converts input stream to String containing lines separated by \n
-     */
-    private String readStream(final InputStream stream) {
-        final BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
-        final StringJoiner builder = new StringJoiner("\n");
-        reader.lines().forEach(builder::add);
-        return builder.toString();
-    }
-}

test/src/main/java/com/github/variety/Variety.java

-package com.github.variety;
-
-import com.github.variety.validator.DbResultsValidator;
-import com.github.variety.validator.JsonResultsValidator;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.DB;
-import com.mongodb.DBCollection;
-import com.mongodb.MongoClient;
-import com.mongodb.ServerAddress;
-
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.StringJoiner;
-
-/**
- * Variety wrapper, provides access to MongoDB database, collection and execution of variety analysis.
- */
-public class Variety {
-
-    /**
-     * Hardcoded database name in variety.js for analysis results
-     */
-    public static final String VARIETY_RESULTS_DBNAME = "varietyResults";
-    public static final String FORMAT_JSON = "json";
-    public static final String FORMAT_ASCII = "ascii";
-
-    public static final String PARAM_QUERY = "query";
-    public static final String PARAM_SORT = "sort";
-    public static final String PARAM_MAXDEPTH = "maxDepth";
-    public static final String PARAM_LIMIT = "limit";
-    public static final String PARAM_OUTPUT_FORMAT = "outputFormat";
-    public static final String PARAM_PERSIST_RESULTS = "persistResults";
-    public static final String PARAM_PLUGINS = "plugins";
-
-
-    private final String inputDatabase;
-    private final String inputCollection;
-    private final MongoClient mongoClient;
-
-    private final Credentials credentials;
-
-    private Integer limit;
-    private Integer maxDepth;
-    private String query;
-    private String sort;
-    private String outputFormat;
-    private boolean quiet;
-    private boolean persistResults;
-    private String[] plugins;
-
-    /**
-     * Create variety wrapper with defined connection do analysed database and collection
-     * @param database   name of database, that will be analysed
-     * @param collection name of collection, that will be analysed
-     * @throws UnknownHostException Thrown when fails connection do default host and port of MongoDB
-     */
-    public Variety(final String database, final String collection) throws UnknownHostException {
-        this(database, collection, null);
-    }
-
-    public Variety(final String inputDatabase, final String inputCollection, final Credentials credentials) throws UnknownHostException {
-        this.inputDatabase = inputDatabase;
-        this.inputCollection = inputCollection;
-        this.credentials = credentials;
-
-        if (credentials == null) {
-            this.mongoClient = new MongoClient();
-        } else {
-            this.mongoClient = new MongoClient(new ServerAddress("localhost"), Arrays.asList(credentials.getMongoCredential()));
-        }
-    }
-
-    /**
-     * @return Access to MongoDB database, where variety stores computed results
-     */
-    public DB getVarietyResultsDatabase() {
-        return mongoClient.getDB(VARIETY_RESULTS_DBNAME);
-    }
-
-    /**
-     * @return Access to collection with source data, that are provided for analysis
-     */
-    public DBCollection getSourceCollection() {
-        return mongoClient.getDB(inputDatabase).getCollection(inputCollection);
-    }
-
-    /**
-     * Variety wrapper for {@code limit} option
-     */
-    public Variety withLimit(final Integer limit) {
-        this.limit = limit;
-        return this;
-    }
-
-    /**
-     * Variety wrapper for {@code maxDepth} option
-     */
-    public Variety withMaxDepth(final Integer maxDepth) {
-        this.maxDepth = maxDepth;
-        return this;
-    }
-
-    /**
-     * Variety wrapper for {@code query} option
-     */
-    public Variety withQuery(final String query) {
-        this.query = query;
-        return this;
-    }
-
-    /**
-     * Variety wrapper for {@code sort} option
-     */
-    public Variety withSort(final String sort) {
-        this.sort = sort;
-        return this;
-    }
-
-    /**
-     * Variety wrapper for {@code format} option.
-     * @param format valid values are either 'json' or 'ascii'
-     */
-    public Variety withFormat(final String format) {
-        this.outputFormat = format;
-        return this;
-    }
-
-    /**
-     * Wrapper for command line option '--quiet', that is passed to mongo shell. Variety is able to read this option
-     * and mute its logs with metadata.
-     */
-    public Variety withQuiet(final boolean quiet) {
-        this.quiet = quiet;
-        return this;
-    }
-
-    /**
-     * Variety wrapper for {@code persistResults} option
-     */
-    public Variety withPersistResults(final boolean persistResults) {
-        this.persistResults = persistResults;
-        return this;
-    }
-
-    public Variety withPlugins(String... plugins) {
-        this.plugins = plugins;
-        return this;
-    }
-
-    /**
-     * Executes mongo shell with configured variety options and variety.js script in path.
-     * @return Stdout of variety.js
-     */
-    public String runAnalysis() throws IOException, InterruptedException {
-        final MongoShell mongoShell = new MongoShell(inputDatabase, credentials, buildParams(), getVarietyPath(), quiet);
-        final String result = mongoShell.execute();
-        System.out.println(result);
-        return result;
-    }
-
-    public ResultsValidator runJsonAnalysis() throws IOException, InterruptedException {
-        final String stdOut = withFormat(FORMAT_JSON).withQuiet(true).runAnalysis();
-        return new JsonResultsValidator(stdOut);
-    }
-
-    public ResultsValidator runDatabaseAnalysis() throws IOException, InterruptedException {
-        final String stdOut = withFormat(FORMAT_ASCII).withPersistResults(true).runAnalysis();
-        return new DbResultsValidator(mongoClient, inputCollection, stdOut);
-    }
-
-    /**
-     * @return Params passed to mongo client together with variety. Collection name is always present, other are optional
-     */
-    private String buildParams() {
-        final StringJoiner args = new StringJoiner(",");
-        args.add("var collection = '" + inputCollection + "'");
-
-        if(limit != null) {
-            args.add(PARAM_LIMIT + " = " + limit);
-        }
-
-        if(maxDepth != null) {
-            args.add(PARAM_MAXDEPTH + " = " + maxDepth);
-        }
-
-        if(query != null && !query.isEmpty()) {
-            args.add(PARAM_QUERY + " = " + query);
-        }
-
-        if(sort != null && !sort.isEmpty()) {
-            args.add(PARAM_SORT + " = " + sort);
-        }
-
-        if(outputFormat != null) {
-            args.add(PARAM_OUTPUT_FORMAT + " = '" + outputFormat + "'");
-        }
-
-        if(persistResults) {
-            args.add(PARAM_PERSIST_RESULTS + " = " + persistResults);
-        }
-
-        if(plugins != null && plugins.length > 0) {
-            args.add(PARAM_PLUGINS + " = \"" + String.join(",", plugins) +  "\"");
-        }
-
-        return args.toString();
-    }
-
-    /**
-     * @return detect absolute path to variety.js, stored in same repository as this tests.
-     */
-    private String getVarietyPath() {
-        // TODO: is there any better way how to compute relative path to variety.js?
-        // relative path from maven compiled classes root to variety.js file.
-        return Paths.get(this.getClass().getResource("/").getFile()).getParent().getParent().getParent().resolve("variety.js").toString();
-    }
-
-
-}

test/src/main/java/com/github/variety/validator/DbResultsValidator.java

-package com.github.variety.validator;
-
-import com.github.variety.Variety;
-import com.mongodb.*;
-import org.junit.Assert;
-
-import java.util.Arrays;
-import java.util.Set;
-
-public class DbResultsValidator implements ResultsValidator {
-
-
-    private final MongoClient mongoClient;
-    private final String sourceCollectionName;
-    private final String stdOut;
-
-    public DbResultsValidator(final MongoClient mongoClient, final String sourceCollectionName, final String stdOut) {
-        this.mongoClient = mongoClient;
-        this.sourceCollectionName = sourceCollectionName;
-        this.stdOut = stdOut;
-    }
-
-    @Override
-    public void validate(final String key, final long totalOccurrences, final double percentContaining, final String... types) {
-        verifyResult(key, totalOccurrences, percentContaining, types);
-    }
-
-    @Override
-    public long getResultsCount() {
-        return getResultsCollection().count();
-    }
-
-    public String getStdOut() {
-        return stdOut;
-    }
-
-    /**
-     * Verifier for collected results in variety analysis
-     * @param key Results should contain entry with this key
-     * @param totalOccurrences Results should contain entry with this total occurrences
-     * @param percentContaining Results should contain entry with this relative percentage
-     * @param types Expected data types of this entry (Based on MongoDB type names)
-     */
-    private void verifyResult(final String key, final long totalOccurrences, final double percentContaining, final String... types) {
-        final DBCursor cursor = getResultsCollection().find(new BasicDBObject("_id.key", key));
-        Assert.assertEquals("Entry with key '" + key + "' not found in variety results", 1, cursor.size());
-        final DBObject result = cursor.next();
-
-        verifyKeyTypes(key, result, types);
-
-        Assert.assertEquals("Failed to verify total occurrences of key " + key, totalOccurrences, ((Double)result.get("totalOccurrences")).longValue());
-        Assert.assertEquals("Failed to verify percents of key " + key, percentContaining, result.get("percentContaining"));
-        cursor.close();
-    }
-
-    private void verifyKeyTypes(final String key, final DBObject result, final String[] expectedTypes) {
-        final BasicDBObject typesObj = (BasicDBObject)((BasicDBObject)result.get("value")).get("types");
-        final Set<String> types = typesObj.keySet();
-
-        Assert.assertEquals(
-                "Incorrect count of expected(" + Arrays.toString(expectedTypes) + ") and real types(" + Arrays.toString(types.toArray())
-                        + ") of key: " + key, expectedTypes.length, types.size());
-
-        for (final String expected : expectedTypes) {
-            if (!types.contains(expected)) {
-                Assert.fail("Type '" + expected + "' not found in real types(" + Arrays.toString(expectedTypes) + ") of key: " + key);
-            }
-        }
-
-    }
-
-    private DBCollection getResultsCollection() {
-        return mongoClient.getDB(Variety.VARIETY_RESULTS_DBNAME).getCollection(getResultsCollectionName());
-    }
-
-    /**
-     * @return name of variety results collection name. Format is {_original_name_}Keys. For collection cars it will be carsKeys.
-     */
-    private String getResultsCollectionName() {
-        return sourceCollectionName + "Keys";
-    }
-}

test/src/main/java/com/github/variety/validator/JsonResultsValidator.java

-package com.github.variety.validator;
-
-import com.mongodb.BasicDBList;
-import com.mongodb.BasicDBObject;
-import com.mongodb.util.JSON;
-import org.junit.Assert;
-
-import java.util.*;
-
-public class JsonResultsValidator implements ResultsValidator {
-
-    private final List<VarietyEntry> entries;
-    private final String stdOut;
-
-    public JsonResultsValidator(final String stdOut) {
-        this.entries = parse(stdOut);
-        this.stdOut = stdOut;
-    }
-
-    private List<VarietyEntry> parse(final String stdOut) {
-        final BasicDBList parse = (BasicDBList) JSON.parse(stdOut);
-        final List<VarietyEntry> entries = new ArrayList<>();
-        for(final Object o : parse) {
-            final BasicDBObject obj = (BasicDBObject)o;
-            final String key = ((BasicDBObject)obj.get("_id")).getString("key");
-            final long totalOccurrences = obj.getLong("totalOccurrences");
-            final double percentContaining = obj.getDouble("percentContaining");
-            final BasicDBObject typesObj = (BasicDBObject) ((BasicDBObject)obj.get("value")).get("types");
-            final Set<String> types = typesObj.keySet();
-            entries.add(new VarietyEntry(key, totalOccurrences, percentContaining, types));
-        }
-        return entries;
-    }
-
-    @Override
-    public void validate(final String key, final long totalOccurrences, final double percentContaining, final String... types) {
-        final Optional<VarietyEntry> first = entries.stream().filter(entry -> entry.getKey().equals(key)).findFirst();
-        if(!first.isPresent()) {
-            Assert.fail("Entry with key '" + key + "' not found in variety results");
-        }
-
-        final VarietyEntry varietyEntry = first.get();
-        Assert.assertEquals("Failed to verify types of key " + key, new HashSet<>(Arrays.asList(types)), varietyEntry.getTypes());
-        Assert.assertEquals("Failed to verify total occurrences of key " + key, totalOccurrences, varietyEntry.getTotalOccurrences());
-        Assert.assertEquals("Failed to verify percents of key " + key, percentContaining, varietyEntry.getPercentContaining(), 1e-15); // TODO: precision?
-    }
-
-    @Override
-    public long getResultsCount() {
-        return entries.size();
-    }
-
-    public String getStdOut() {
-        return stdOut;
-    }
-
-    private class VarietyEntry {
-        private final String key;
-        private final long totalOccurrences;
-        private final double percentContaining;
-        private final Set<String> types;
-
-        private VarietyEntry(final String key, final long totalOccurrences, final double percentContaining, final Set<String> types) {
-            this.key = key;
-            this.totalOccurrences = totalOccurrences;
-            this.percentContaining = percentContaining;
-            this.types = types;
-        }
-
-        private String getKey() {
-            return key;
-        }
-
-        private long getTotalOccurrences() {
-            return totalOccurrences;
-        }
-
-        private double getPercentContaining() {
-            return percentContaining;
-        }
-
-        private Set<String> getTypes() {
-            return types;
-        }
-    }
-}

test/src/main/java/com/github/variety/validator/ResultsValidator.java

-package com.github.variety.validator;
-
-public interface ResultsValidator {
-    void validate(String key, long totalOccurrences, double percentContaining, String... types);
-    long getResultsCount();
-    String getStdOut();
-}

test/src/test/java/com/github/variety/test/BasicAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests basic collection structure provided in readme of variety
- */
-public class BasicAnalysisTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    /**
-     * Validate correct results read from DB
-     */
-    @Test
-    public void verifyBasicResultsDb() throws Exception {
-        validate(variety.runDatabaseAnalysis());
-    }
-
-    /**
-     * Validate correct results read from JSON standard output
-     */
-    @Test
-    public void verifyBasicResultsJson() throws Exception {
-        validate(variety.runJsonAnalysis());
-    }
-
-    private void validate(final ResultsValidator analysis) {
-        analysis.validate("_id", 5, 100, "ObjectId");
-        analysis.validate("name", 5, 100, "String");
-        analysis.validate("bio", 3, 60, "String");
-        analysis.validate("pets", 2, 40, "String", "Array");
-        analysis.validate("someBinData", 1, 20, "BinData-old");
-        analysis.validate("someWeirdLegacyKey", 1, 20, "String");
-    }
-}

test/src/test/java/com/github/variety/test/DatatypeRecognitionTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.BasicDBObject;
-import org.bson.types.Binary;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Date;
-
-/**
- * Verify, that variety can recognize all usual datatypes, including different bindata types.
- * This test addresses issue https://github.com/variety/variety/issues/8
- */
-public class DatatypeRecognitionTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(new BasicDBObject()
-            .append("key_string", "Just plain String")
-            .append("key_boolean", true)
-            .append("key_number", 1)
-            .append("key_date", new Date())
-            .append("key_binData-generic", new Binary((byte)0x00, new byte[]{1,2,3,4}))
-            .append("key_binData-function", new Binary((byte) 0x01, new byte[]{1,2,3,4}))
-            .append("key_binData-old", new Binary((byte) 0x02, new byte[]{1,2,3,4}))
-            .append("key_binData-UUID", new Binary((byte) 0x03, new byte[]{1,2,3,4}))
-            .append("key_binData-MD5", new Binary((byte) 0x05, new byte[]{1,2,3,4}))
-            .append("key_binData-user", new Binary((byte) 0x80, new byte[]{1,2,3,4}))
-            .append("key_array", new ArrayList<>())
-            .append("key_object", new BasicDBObject())
-            .append("key_null", null)
-        );
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void testDatatypeRecognition() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis();
-
-        Assert.assertEquals(14, analysis.getResultsCount());
-
-        analysis.validate("_id", 1, 100, "ObjectId");
-        analysis.validate("key_string", 1, 100, "String");
-        analysis.validate("key_boolean", 1, 100, "Boolean");
-        analysis.validate("key_number", 1, 100, "Number");
-        analysis.validate("key_date", 1, 100, "Date");
-        analysis.validate("key_binData-generic", 1, 100, "BinData-generic");
-        analysis.validate("key_binData-function", 1, 100, "BinData-function");
-        analysis.validate("key_binData-old", 1, 100, "BinData-old");
-        analysis.validate("key_binData-UUID", 1, 100, "BinData-UUID");
-        analysis.validate("key_binData-MD5", 1, 100, "BinData-MD5");
-        analysis.validate("key_binData-user", 1, 100, "BinData-user");
-        analysis.validate("key_array", 1, 100, "Array");
-        analysis.validate("key_object", 1, 100, "Object");
-        analysis.validate("key_null", 1, 100, "null"); // TODO: why has 'null' first letter lowercase, unlike all other types?
-    }
-}

test/src/test/java/com/github/variety/test/LimitResultsAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.DBObject;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests limit functionality of variety. It should analyse only first _n_ objects.
- */
-public class LimitResultsAnalysisTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void verifyLimitedResults() throws Exception {
-        final ResultsValidator analysis = variety.withLimit(1).runDatabaseAnalysis();
-        // limit=1 without other params selects the last inserted document (see SampleData class)
-        // it should equals {name: "Jim", someBinData: new BinData(2,"1234")}
-        analysis.validate("_id", 1, 100, "ObjectId");
-        analysis.validate("name", 1, 100, "String");
-        analysis.validate("someBinData", 1, 100, "BinData-old");
-    }
-
-    @Test
-    public void verifyLimitOverMaxResults() throws Exception {
-
-        // limit is set to higher number, that the actual number of documents in collection
-        // analysis should compute percentages based on the real number of documents, not on the
-        // number provided in the limit var.
-        final ResultsValidator analysis = variety.withLimit(10).runDatabaseAnalysis();
-
-        analysis.validate("_id", 5, 100, "ObjectId");
-        analysis.validate("name", 5, 100, "String");
-        analysis.validate("bio", 3, 60, "String");
-        analysis.validate("pets", 2, 40, "String", "Array");
-        analysis.validate("someBinData", 1, 20, "BinData-old");
-        analysis.validate("someWeirdLegacyKey", 1, 20, "String");
-    }
-}

test/src/test/java/com/github/variety/test/LimitedAccessTest.java

-package com.github.variety.test;
-
-import com.github.variety.Credentials;
-import com.github.variety.MongoShell;
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.MongoClient;
-import com.mongodb.ServerAddress;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Tests, if variety can return results for user with read only access to analyzed database (without permission to list
- * all other dbs / collections, without permission to persist results).
- */
-@Ignore
-public class LimitedAccessTest {
-
-    private Variety variety;
-    private MongoClient adminConnection;
-
-    @Before
-    public void setUp() throws Exception {
-
-        // create admin user (expects empty users table => no auth used for this connection)
-        createUser(null, Credentials.ADMIN);
-
-        // create limited user
-        createUser(Credentials.ADMIN, Credentials.USER);
-
-        // connect with admin credentials
-        adminConnection = new MongoClient(new ServerAddress("localhost"), Arrays.asList(Credentials.ADMIN.getMongoCredential()));
-
-        // create sample collection (logged admin user, writes to test DB)
-        adminConnection.getDB("test").getCollection("users").insert(SampleData.getDocuments());
-
-        // initialize variety with limited user credentials, connects to test/users collection
-        variety = new Variety("test", "users", Credentials.USER);
-    }
-
-    private void createUser(final Credentials loginCredentials, final Credentials userToCreate) throws IOException, InterruptedException {
-        final MongoShell shell = new MongoShell(userToCreate.getAuthDatabase(), loginCredentials, "db.addUser(" + userToCreate.getUserDocument() + ")", null, false);
-       System.out.println(shell.execute());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        adminConnection.getDB("test").getCollection("users").drop();
-
-        // remove both users from admin (auth) database. Caution, order is important - first delete user, then admin
-        System.out.println(new MongoShell("test", Credentials.ADMIN, "db.removeUser('" + Credentials.USER.getUsername() + "')", null, false).execute());
-        System.out.println(new MongoShell("admin", Credentials.ADMIN, "db.removeUser('" + Credentials.ADMIN.getUsername() + "')", null, false).execute());
-    }
-
-    /**
-     * Validate correct results read from JSON standard output, limited user connection provided
-     */
-    @Test
-    public void verifyBasicResultsJson() throws Exception {
-        validate(variety.runJsonAnalysis());
-    }
-
-    @Test
-    public void verifyBasicResultsAscii() throws Exception {
-        final String stdout = variety.withPersistResults(false).withQuiet(true).runAnalysis();
-        Assert.assertEquals(SampleData.getExpectedDataAsciiTable(), stdout);
-    }
-
-
-    @Test
-    public void testNotFoundDatabaseForAdmin() throws Exception {
-        final Variety adminVariety = new Variety("foo", "users", Credentials.ADMIN);
-        try {
-            adminVariety.runAnalysis();
-            Assert.fail("Should throw exception");
-        } catch (final Exception e) {
-            System.out.println(e);
-            final String messageVersion24 = "The collection specified (users) in the database specified (foo) does not exist or is empty";
-            final String messageVersion26 = "The database specified (foo) does not exist";
-            Assert.assertTrue(e.getMessage().contains(messageVersion24) || e.getMessage().contains(messageVersion26));
-        }
-    }
-
-    @Test
-    public void testNotFoundCollectionForAdmin() throws Exception {
-        final Variety adminVariety = new Variety("test", "bar", Credentials.ADMIN);
-        try {
-            adminVariety.runAnalysis();
-            Assert.fail("Should throw exception");
-        } catch (final Exception e) {
-            Assert.assertTrue(e.getMessage().contains("The collection specified (bar) in the database specified (test) does not exist or is empty."));
-            Assert.assertTrue(e.getMessage().contains("Possible collection options for database specified:"));
-        }
-    }
-
-    @Test
-    public void testNotFoundCollectionForUser() throws Exception {
-        final Variety adminVariety = new Variety("test", "bar", Credentials.USER);
-        try {
-            adminVariety.runAnalysis();
-            Assert.fail("Should throw exception");
-        } catch (final Exception e) {
-            Assert.assertTrue(e.getMessage().contains("The collection specified (bar) in the database specified (test) does not exist or is empty."));
-            Assert.assertTrue(e.getMessage().contains("Possible collection options for database specified:"));
-        }
-    }
-
-    @Test
-    public void testNotFoundDbForUser() throws Exception {
-        final Variety adminVariety = new Variety("foo", "users", Credentials.USER);
-        try {
-            adminVariety.runAnalysis();
-            Assert.fail("Should throw exception");
-        } catch (final Exception e) {
-            Assert.assertTrue("Exception should contain info about not authorized access, full message is: '" + e.getMessage() + "'", e.getMessage().contains("not authorized"));
-        }
-    }
-
-    private void validate(final ResultsValidator analysis) {
-        analysis.validate("_id", 5, 100, "ObjectId");
-        analysis.validate("name", 5, 100, "String");
-        analysis.validate("bio", 3, 60, "String");
-        analysis.validate("pets", 2, 40, "String", "Array");
-        analysis.validate("someBinData", 1, 20, "BinData-old");
-        analysis.validate("someWeirdLegacyKey", 1, 20, "String");
-    }
-}

test/src/test/java/com/github/variety/test/MaxDepthAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.DBObject;
-import com.mongodb.util.JSON;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class MaxDepthAnalysisTest {
-
-    private static final double EXPECTED_PERCENTS = 100;
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        variety = new Variety("test", "users");
-        variety.getSourceCollection().insert((DBObject) JSON.parse("{name:'Walter', someNestedObject:{a:{b:{c:{d:{e:1}}}}}})"));
-
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void testUnlimitedAnalysis() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis();
-        Assert.assertEquals("Variety results have not correct count of entries", 8, analysis.getResultsCount()); // 8 results, including '_id' and 'name'
-
-        analysis.validate("_id", 1, EXPECTED_PERCENTS, "ObjectId");
-        analysis.validate("name", 1, EXPECTED_PERCENTS, "String");
-
-        analysis.validate("someNestedObject", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a.b", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a.b.c", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a.b.c.d", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a.b.c.d.e", 1, EXPECTED_PERCENTS, "Number");
-    }
-
-    @Test
-    public void testLimitedDepthAnalysis() throws Exception {
-        final ResultsValidator analysis = variety.withMaxDepth(3).runDatabaseAnalysis();
-
-        Assert.assertEquals("Variety results have not correct count of entries", 5, analysis.getResultsCount()); // 5 results, including '_id' and 'name'
-
-        analysis.validate("_id", 1, EXPECTED_PERCENTS, "ObjectId");
-        analysis.validate("name", 1, EXPECTED_PERCENTS, "String");
-
-        analysis.validate("someNestedObject", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a", 1, EXPECTED_PERCENTS, "Object");
-        analysis.validate("someNestedObject.a.b", 1, EXPECTED_PERCENTS, "Object");
-    }
-
-
-}

test/src/test/java/com/github/variety/test/OutputFormatTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class OutputFormatTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void verifyJsonEntries() throws Exception {
-        final ResultsValidator analysis = variety
-                .withQuiet(true) // do not output any other metadata, only results
-                .withFormat(Variety.FORMAT_JSON)
-                .runJsonAnalysis();
-
-        // there should be seven different json results
-        Assert.assertEquals(7, analysis.getResultsCount());
-    }
-
-    @Test
-    public void verifyAsciiTableOutput() throws Exception {
-        final ResultsValidator analysis = variety.withFormat(Variety.FORMAT_ASCII).runDatabaseAnalysis();
-
-        // filter only lines starting with character '|'
-        final String actual = Stream.of(analysis.getStdOut().split("\n"))
-                .filter(line -> line.startsWith("|") || line.startsWith("+"))
-                .collect(Collectors.joining("\n"));
-
-        Assert.assertEquals(SampleData.getExpectedDataAsciiTable(), actual);
-
-    }
-}

test/src/test/java/com/github/variety/test/ParametersParsingTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Verify, that variety can read and use(re-print to stdout) passed parameters like limit, sort, query and maxDepth.
- */
-public class ParametersParsingTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    /**
-     * Verify default parameters of variety.
-     */
-    @Test
-    public void verifyDefaultResultsStdout() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis();
-
-        final Map<String, String> params = getParamsMap(analysis.getStdOut());
-
-        Assert.assertEquals("99", params.get(Variety.PARAM_MAXDEPTH));
-        Assert.assertEquals("{ }", params.get(Variety.PARAM_QUERY));
-        Assert.assertEquals("{ \"_id\" : -1 }", params.get(Variety.PARAM_SORT));
-        Assert.assertEquals("5", params.get(Variety.PARAM_LIMIT)); // TODO: why is limit configured to current count, not set as 'unlimited'? It could save one count query
-    }
-
-    /**
-     * Verify, that all passed parameters are correctly recognized and printed out in stdout of variety.
-     */
-    @Test
-    public void verifyRestrictedResultsStdout() throws Exception {
-        final ResultsValidator analysis = variety
-                .withQuery("{name:'Harry'}")
-                .withSort("{name:1}")
-                .withMaxDepth(5)
-                .withLimit(2)
-                .runDatabaseAnalysis();
-
-        final Map<String, String> params = getParamsMap(analysis.getStdOut());
-
-        Assert.assertEquals("5", params.get(Variety.PARAM_MAXDEPTH));
-        Assert.assertEquals("{ \"name\" : \"Harry\" }", params.get(Variety.PARAM_QUERY));
-        Assert.assertEquals("{ \"name\" : 1 }", params.get(Variety.PARAM_SORT));
-        Assert.assertEquals("2", params.get(Variety.PARAM_LIMIT));
-    }
-
-    /**
-     * Verify, that variety recognizes unknown or empty collection and exists. In stdout should be recorded reason.
-     */
-    @Test
-    public void testUnknownCollectionResponse() throws Exception {
-        this.variety = new Variety("test", "--unknown--");
-        try {
-            variety.runDatabaseAnalysis();
-            Assert.fail("It should throw exception");
-        } catch (final RuntimeException e) {
-            Assert.assertTrue(e.getMessage().contains("does not exist or is empty"));
-        }
-    }
-
-    @Test
-    public void testDefaultOutputFormatParam() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis(); // format option not provided
-        final Map<String, String> params = getParamsMap(analysis.getStdOut());
-        Assert.assertEquals("\"ascii\"", params.get(Variety.PARAM_OUTPUT_FORMAT));
-    }
-
-    @Test
-    public void testAsciiOutputFormatParam() throws Exception {
-        final ResultsValidator analysis = variety.withFormat(Variety.FORMAT_ASCII).runDatabaseAnalysis();
-        final Map<String, String> params = getParamsMap(analysis.getStdOut());
-        Assert.assertEquals("\"ascii\"", params.get(Variety.PARAM_OUTPUT_FORMAT));
-    }
-
-    @Test
-    public void testPersistResultsParam() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis();
-        final Map<String, String> params = getParamsMap(analysis.getStdOut());
-        Assert.assertEquals("true", params.get(Variety.PARAM_PERSIST_RESULTS));
-    }
-
-    @Test
-    public void testJsonOutputFormatParam() throws Exception {
-        final ResultsValidator analysis = variety.withFormat(Variety.FORMAT_JSON).runJsonAnalysis();
-        // verify, that result is clean parsable json with 7 entries found
-        Assert.assertEquals(7, analysis.getResultsCount());
-    }
-
-    /**
-     * @param stdout Text from mongo shell, containing variety config output + json results
-     * @return Map of config values
-     */
-    private Map<String, String> getParamsMap(final String stdout) {
-        return Stream.of(stdout.split("\n"))
-                .filter(line -> line.startsWith("Using "))
-                .map(v -> v.replace("Using ", ""))
-                .collect(Collectors.toMap(k -> k.split(" of ")[0], v -> v.split(" of ")[1]));
-    }
-}

test/src/test/java/com/github/variety/test/PluginsTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import jdk.nashorn.internal.runtime.regexp.joni.Regex;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.net.URL;
-import java.util.regex.Pattern;
-
-public class PluginsTest {
-
-    private Variety variety;
-
-    public static final String EXPECTED_OUTPUT =
-            "key|types|occurrences|percents\n" +
-            "_id|ObjectId|5|100\n" +
-            "name|String|5|100\n" +
-            "bio|String|3|60\n" +
-            "birthday|String|2|40\n" +
-            "pets|Array,String|2|40\n" +
-            "someBinData|BinData-old|1|20\n" +
-            "someWeirdLegacyKey|String|1|20";
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    /**
-     * Validate correct results read from DB
-     */
-    @Test
-    public void verifyFormatResults() throws Exception {
-        final String path = getPluginPath("/csvplugin.js");
-        final ResultsValidator analysis = variety.withQuiet(true).withPlugins(path).runDatabaseAnalysis();
-        Assert.assertEquals(EXPECTED_OUTPUT, analysis.getStdOut());
-    }
-
-    @Test
-    public void verifyPluginParamParsing() throws Exception {
-        final String path = getPluginPath("/csvplugin.js");
-        final ResultsValidator analysis = variety.withPlugins(path + "|delimiter=;").runDatabaseAnalysis();
-        Assert.assertTrue(analysis.getStdOut().contains(path));
-    }
-
-    private String getPluginPath(final String name) {
-        final URL resource = this.getClass().getResource(name);
-        return resource.getFile();
-    }
-}

test/src/test/java/com/github/variety/test/QueryLimitedAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class QueryLimitedAnalysisTest {
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void testQueryLimitedAnalysis() throws Exception {
-        final ResultsValidator analysis = variety.withQuery("{birthday:{$exists: true}}").runDatabaseAnalysis();
-        Assert.assertEquals(5, analysis.getResultsCount());
-
-        analysis.validate("_id", 2, 100, "ObjectId");
-        analysis.validate("birthday", 2, 100, "String");
-        analysis.validate("name", 2, 100, "String");
-        analysis.validate("bio", 1, 50, "String");
-        analysis.validate("pets", 1, 50, "String");
-
-
-    }
-}

test/src/test/java/com/github/variety/test/QuietOptionTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Variety can read '--quiet' option passed to mongo shell and mute all debug/metadata logs. In this case only
- * results should be printed. Together with output format set to json should be possible to simply forward output
- * from variety to another tool processing valid json.
- */
-public class QuietOptionTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    /**
-     * verify, that output contains only results table and nothing more
-     */
-    @Test
-    public void testQuietLogs() throws Exception {
-        final ResultsValidator varietyAnalysis = variety.withQuiet(true).runDatabaseAnalysis();
-        Assert.assertEquals(SampleData.getExpectedDataAsciiTable(), varietyAnalysis.getStdOut());
-    }
-}

test/src/test/java/com/github/variety/test/SampleData.java

-package com.github.variety.test;
-
-import com.mongodb.BasicDBObjectBuilder;
-import com.mongodb.DBObject;
-import org.bson.types.Binary;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-class SampleData {
-
-    /**
-     * Java representation of sample collection provided in variety README:<p>
-     *
-     * {name: "Tom", bio: "A nice guy.", pets: ["monkey", "fish"], someWeirdLegacyKey: "I like Ike!"}<p>
-     * {name: "Dick", bio: "I swordfight.", birthday: new Date("1974/03/14")}<p>
-     * {name: "Harry", pets: "egret", birthday: new Date("1984/03/14")}<p>
-     * {name: "Geneviève", bio: "Ça va?"}<p>
-     * {name: "Jim", someBinData: new BinData(2,"1234")}<p>
-     */
-    public static List<DBObject> getDocuments() {
-        final List<DBObject> examples = new ArrayList<>();
-        examples.add(
-            new BasicDBObjectBuilder()
-                .add("name", "Tom")
-                .add("bio", "A nice guy.")
-                .add("pets", Arrays.asList("monkey", "fish"))
-                .add("someWeirdLegacyKey", "I like Ike!")
-            .get()
-        );
-        examples.add(
-            new BasicDBObjectBuilder()
-                .add("name", "Dick")
-                .add("bio", "I swordfight.")
-                .add("birthday", LocalDate.of(1974, 3, 14).toString())
-            .get()
-        );
-        examples.add(
-            new BasicDBObjectBuilder()
-                .add("name", "Harry")
-                .add("pets", "egret")
-                .add("birthday", LocalDate.of(1984, 3, 14).toString())
-            .get()
-        );
-        examples.add(
-            new BasicDBObjectBuilder()
-                .add("name", "Geneviève")
-                .add("bio", "Ça va?")
-            .get()
-        );
-        examples.add(
-            new BasicDBObjectBuilder()
-                .add("name", "Jim")
-                .add("someBinData", new Binary((byte) 0x02, new byte[]{1,2,3,4}))
-            .get()
-        );
-
-        return examples;
-    }
-
-    /**
-     * Ascii table representation of sample data results. It should be possible to verify actual output of Variety
-     * against this table, to check correct formatting.
-     */
-    public static String getExpectedDataAsciiTable() {
-        try {
-            return new String(Files.readAllBytes(Paths.get(SampleData.class.getResource("/expected_ascii_table.txt").getFile())));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

test/src/test/java/com/github/variety/test/SortedAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Verify, that variety can handle sort parameter and analyse collection in given order. It is useful only when
- * used together with limit.
- */
-public class SortedAnalysisTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(SampleData.getDocuments());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void testSortedAnalysis() throws Exception {
-        // Sort without limit or other query should not modify results itself. Analysis is done on the same data, only in another order.
-        final ResultsValidator analysis = variety.withSort("{name:-1}").runDatabaseAnalysis();
-        analysis.validate("_id", 5, 100, "ObjectId");
-        analysis.validate("name", 5, 100, "String");
-        analysis.validate("bio", 3, 60, "String");
-        analysis.validate("pets", 2, 40, "String", "Array");
-        analysis.validate("someBinData", 1, 20, "BinData-old");
-        analysis.validate("someWeirdLegacyKey", 1, 20, "String");
-
-    }
-
-    @Test
-    public void testSortedAnalysisWithLimit() throws Exception {
-        // when sorting default SampleData by name desc, first entry becomes Tom. He is only with key 'someWeirdLegacyKey'
-        // Together with applying limit 1, Tom is the only result in analysis. That gives us chance to assume keys and verify
-        // that ordering is correct.
-        // {name: "Tom", bio: "A nice guy.", pets: ["monkey", "fish"], someWeirdLegacyKey: "I like Ike!"}
-        final ResultsValidator analysis = variety.withSort("{name:-1}").withLimit(1).runDatabaseAnalysis();
-
-        Assert.assertEquals(5, analysis.getResultsCount());
-
-        analysis.validate("_id", 1, 100, "ObjectId");
-        analysis.validate("name", 1, 100, "String");
-        analysis.validate("bio", 1, 100, "String");
-        analysis.validate("pets", 1, 100, "Array");
-        analysis.validate("someWeirdLegacyKey", 1, 100, "String");
-
-    }
-}

test/src/test/java/com/github/variety/test/UnnamedObjectsAnalysisTest.java

-package com.github.variety.test;
-
-import com.github.variety.Variety;
-import com.github.variety.validator.ResultsValidator;
-import com.mongodb.DBObject;
-import com.mongodb.util.JSON;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-/**
- * Test, how variety handles objects, that are not named (for example objects inside array).
- * It addresses behavior described in issue https://github.com/variety/variety/issues/29
- */
-public class UnnamedObjectsAnalysisTest {
-
-    private Variety variety;
-
-    @Before
-    public void setUp() throws Exception {
-        this.variety = new Variety("test", "users");
-        variety.getSourceCollection().insert(Arrays.asList(
-                createDbObj("{title:'Article 1', comments:[{author:'John', body:'it works', visible:true }]}"),
-                createDbObj("{title:'Article 2', comments:[{author:'Tom', body:'thanks'}, {author:'Mark', body:1}]}")
-        ));
-    }
-
-    private DBObject createDbObj(final String json) {
-        return (DBObject) JSON.parse(json);
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        variety.getVarietyResultsDatabase().dropDatabase();
-        variety.getSourceCollection().drop();
-    }
-
-    @Test
-    public void testUnnamedObjects() throws Exception {
-        final ResultsValidator analysis = variety.runDatabaseAnalysis();
-
-        Assert.assertEquals(6, analysis.getResultsCount());
-
-        analysis.validate("_id", 2, 100, "ObjectId");
-        analysis.validate("title", 2, 100, "String");
-        analysis.validate("comments", 2, 100, "Array");
-
-        // unnamed objects are prefixed with .XX key
-        analysis.validate("comments.XX.author", 2, 100, "String");
-        analysis.validate("comments.XX.body", 2, 100, "String", "Number");
-        analysis.validate("comments.XX.visible", 1, 50, "Boolean");
-    }
-}

test/src/test/java/com/github/variety/test/VersionInfoTest.java

-package com.github.variety.test;
-
-import junit.framework.AssertionFailedError;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Compare, that variety version info is identical in variety.js and in CHANGELOG files.
- */
-public class VersionInfoTest {
-
-    private static final Pattern VARIETYJS_PATTERN = Pattern.compile("\\w+\\('(.+), released (.+)'\\).*");
-    private static final Pattern CHANGELOG_PATTERN = Pattern.compile("\\((.+)\\)(.+):(.*)");
-    
-    private List<String> varietyLines;
-    private List<String> changelogLines;
-
-    @Before
-    public void setUp() throws Exception {
-        varietyLines = Files.readAllLines(getFile("variety.js"));
-        changelogLines = Files.readAllLines(getFile("CHANGELOG"));
-    }
-
-    @Test
-    public void testVersionsEquality() throws Exception {
-        Assert.assertEquals("Version provided in variety.js is different from given in CHANGELOG",
-                getChangelogVersion(changelogLines), getVarietyVersion(varietyLines));
-    }
-
-    @Test
-    public void testDatesEquality() throws Exception {
-        Assert.assertEquals("Date provided in variety.js is different from given in CHANGELOG",
-                getChangelogDate(changelogLines), getVarietyDate(varietyLines));
-    }
-
-    private String getVarietyVersion(final List<String> variety) {
-        return getVarietyPatternGroup(variety, 1);
-    }
-
-    private String getVarietyDate(final List<String> variety) {
-        return getVarietyPatternGroup(variety, 2);
-    }
-
-    private String getChangelogDate(final List<String> changelog) {
-        return getChangelogPatternGroup(changelog, 1);
-    }
-
-    private String getChangelogVersion(final List<String> changelog) {
-        return getChangelogPatternGroup(changelog, 2);
-    }
-
-    private String getVarietyPatternGroup(final List<String> variety, final int group) {
-        for (final String line : variety) {
-            final Matcher matcher = VARIETYJS_PATTERN.matcher(line);
-            if (matcher.matches()) {
-                return matcher.group(group);
-            }
-        }
-        throw new AssertionFailedError("Variety.js does not contain version and date info");
-    }
-
-    private String getChangelogPatternGroup(final List<String> changelog, final int group) {
-        final Matcher matcher = CHANGELOG_PATTERN.matcher(changelog.get(0));
-        if (!matcher.find()) {
-            throw new AssertionFailedError("CHANGELOG does not contain version and date info");
-        }
-        return matcher.group(group).trim();
-    }
-
-    private Path getFile(final String filename) {
-        // on linux could it be for example /{path_to_project}/variety/test/target/test-classes
-        final String testClassesPath = this.getClass().getResource("/").getFile();
-
-        // traverse from test classes path to variety base directory
-        return Paths.get(testClassesPath).getParent().getParent().getParent().resolve(filename);
-    }
-}