Commit 5392f944 by James Cropcho

getting better all the time

parent 629874d1
if (typeof collection == "undefined") { if (typeof collection == "undefined") {
throw "You have to supply a 'collection' variable, a la \"--eval 'var limit = 10'\""; throw "You have to supply a 'collection' variable, a la \"--eval 'var limit = 10'\"";
} }
//db.system.js.save( { _id : "getCollection", value : function() { return eval(collection); }} );
if (typeof limit == "undefined") { limit = db[collection].count(); } if (typeof limit == "undefined") { limit = db[collection].count(); }
print("Using limit of " + limit); print("Using limit of " + limit);
//var alreadyEmitted = new Array(); // global
//schemaAnalyzerKeyIsPresent = function(toBeEmitted, alreadyEmitted) {
// for(key in alreadyEmitted) {
// if(key["key"] === toBeEmitted["key"] && key["type"] === toBeEmitted["type"]) {
// return true;
// }
// }
// return false;
//}
//db.system.js.save( { _id : "schemaAnalyzerKeyIsPresent", value : schemaAnalyzerKeyIsPresent } );
schemaAnalyzerCanHaveChildren = function (v) { schemaAnalyzerCanHaveChildren = function (v) {
var isArray = v && var isArray = v &&
typeof v === 'object' && typeof v === 'object' &&
...@@ -34,12 +21,7 @@ schemaAnalyzerMapRecursive = function(parentKey, keys) { ...@@ -34,12 +21,7 @@ schemaAnalyzerMapRecursive = function(parentKey, keys) {
key = (parentKey + "." + key).replace(/\.\d+/g,'.XX'); key = (parentKey + "." + key).replace(/\.\d+/g,'.XX');
toBeEmitted = {key: key, type: typeof value}; emit({key : key}, {type: schemaAnalyzerTypeOf(value)});
// if(!schemaAnalyzerKeyIsPresent(toBeEmitted,alreadyEmitted)) {
emit(toBeEmitted, null);
// alreadyEmitted.push(toBeEmitted);
// }
if (schemaAnalyzerCanHaveChildren(value)) { if (schemaAnalyzerCanHaveChildren(value)) {
schemaAnalyzerMapRecursive(key, value); schemaAnalyzerMapRecursive(key, value);
...@@ -48,22 +30,35 @@ schemaAnalyzerMapRecursive = function(parentKey, keys) { ...@@ -48,22 +30,35 @@ schemaAnalyzerMapRecursive = function(parentKey, keys) {
} }
db.system.js.save({_id: "schemaAnalyzerMapRecursive", value: schemaAnalyzerMapRecursive}); db.system.js.save({_id: "schemaAnalyzerMapRecursive", value: schemaAnalyzerMapRecursive});
schemaAnalyzerTypeOf = function(thing) {
var nativeType = typeof thing;
if (!thing || thing === null) { return "null"; }
if (nativeType !== "object") {
return nativeType;
}
else {
if (thing && thing.constructor === Array) {
return "array";
}
else {
return "object";
}
}
}
db.system.js.save({_id: "schemaAnalyzerTypeOf", value: schemaAnalyzerTypeOf});
map = function() { map = function() {
var keys = this; var keys = this;
// var alreadyEmitted = new Array(); // reset global variable
for (var key in keys) { for (var key in keys) {
var value = keys[key]; var value = keys[key];
key = key.replace(/\.\d+/g,'.XX'); key = key.replace(/\.\d+/g,'.XX');
toBeEmitted = {key : key, type : typeof value}; emit({key : key}, {type: schemaAnalyzerTypeOf(value)});
// if(!schemaAnalyzerKeyIsPresent(toBeEmitted,alreadyEmitted)) {
// emit(toBeEmitted, {occurrences: 1});
emit(toBeEmitted, null);
// alreadyEmitted.push(toBeEmitted);
// }
if (schemaAnalyzerCanHaveChildren(value)) { if (schemaAnalyzerCanHaveChildren(value)) {
schemaAnalyzerMapRecursive(key, value); schemaAnalyzerMapRecursive(key, value);
...@@ -72,32 +67,19 @@ map = function() { ...@@ -72,32 +67,19 @@ map = function() {
} }
reduce = function(key, values){ reduce = function(key, values){
// var occurrences = 0; var types = [];
// values.forEach(function(value) { values.forEach(function(value) {
// occurrences += value.occurrences; if(types.indexOf(value.type) === -1) {
// }); types.push(value.type);
}
});
// return {occurrences: occurrences}; return { types: types };
return null;
} }
//finalize = function(key, value) {
// value.percentage = value.occurrences / limit * 100.0;
// return value;
//}
//finalize = function(key, value) {
// if(!key["key"].match(/\.XX/)) {
// value = collectionDB.count({key: {$exists: true}});
// }
// return value;
//}
var resultsCollectionName = collection + "Keys"; var resultsCollectionName = collection + "Keys";
//resultsDB[resultsCollectionName].ensureIndex({ "_id.key" : 1 , "_id.type" : 1},{unique : true}); db[collection].mapReduce(map, reduce, {
db[collection].mapReduce(map, reduce, { //finalize: finalize,
out: { out: {
replace : resultsCollectionName, replace : resultsCollectionName,
db : "schemaAnalyzerResults"}, db : "schemaAnalyzerResults"},
...@@ -107,30 +89,29 @@ db[collection].mapReduce(map, reduce, { //finalize: finalize, ...@@ -107,30 +89,29 @@ db[collection].mapReduce(map, reduce, { //finalize: finalize,
var resultsDB = db.getMongo().getDB("schemaAnalyzerResults"); var resultsDB = db.getMongo().getDB("schemaAnalyzerResults");
var keys = resultsDB[resultsCollectionName].find(); var numDocuments = db[collection].count();
resultsDB[resultsCollectionName].find({}).forEach(function(key) {
keyName = key["_id"].key;
keys.forEach(function(key) { if(keyName.match(/\.XX$/)) {
delete key.value; resultsDB[resultsCollectionName].remove({ "_id" : key["_id"]});
keyName = key["_id"].key; // should be keys?? return;
}
var existsQuery = {}; var existsQuery = {};
existsQuery[keyName] = {$exists: true}; existsQuery[keyName] = {$exists: true};
if(!keyName.match(/\.XX/)) { key.totalOccurrences = db[collection].count(existsQuery);
key.totalOccurences = db[collection].count(existsQuery); key.percentContaining = (key.totalOccurrences / numDocuments) * 100
}
else {
var sizeQuery = {};
sizeQuery[keyName] = {$size: true};
keyName.totalOccurences = db[collection].count(existsQuery) - db[collection].count(sizeQuery);
}
// delete key.value; // created by our empty reduce()
resultsDB[resultsCollectionName].save(key); resultsDB[resultsCollectionName].save(key);
print(tojson(key, '', true));
}); });
//> db.users.count({"smartalert.daily": {$exists: true}}); var sortedKeys = resultsDB[resultsCollectionName].find({}).sort({totalOccurrences: -1});
//23
//> db.users.count({"smartalert.daily": {$size: 0}}); sortedKeys.forEach(function(key) {
//3 print(tojson(key, '', true));
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment