Diff for Redis between two databases - redis

Question

What is the best way to see differences between the contents of two different Redis databases? We have a development and production deployment and our production instance doesn't seem to have the same exact data as development but we need a solid easy way to test that - is there a good way to do this without writing too much of our own code?

Answer 1

If your production data has a high rate of change, doing this would be tough. So for this answer let us assume you either don't have a high data churn or you can do this at a quiescent time where the data churn rate is low.
For this to work from a shell script perspective you will need to do the first task in parallel.
Use the RDB save option in redis-cli to pull down a local copy of the data for each server.
Compare a hash of the files, such as an md5sum. If they are the same, the data is the same.
If the churn rate is low enough that you can get a comparable dump, this will identify if they are different, but not what data differs if any. However you can use this as a conditional for avoiding a deeper dive which has to essentially ransack the database.
Alternatively, if they differ you could write something which uses the RDB files and one of the RDB file parsers to compare them and output the differences it finds.
Indeed this method would work far better than ransacking the server as data can change during the process whereas the dump file analysis method compares for a fixed point in time.

Answer 2

Because we are using Redis Sentinel I assumed I couldn't find some code to steal, so wrote my own real quick
the code compares the contents of two different Redis databases, using the ioredis Redis client NPM module. The code works fine for small Redis databases that don't change much, for larger amounts of data or for databases that are seeing a lot of writes per minute, this might not be suitable.
code looks like this:
var async = require('async');
var Redis = require('ioredis');
var REDIS_SENTINEL_MASTER_NAME_STAGING = 'smartconndevredis01';
var REDIS_SENTINEL_MASTER_NAME_PROD = 'smartconnect_curr_master';
var SENTINEL_ENDPOINTS_STAGING = [
{
"host": "XXX.XX.XX.XX",
"port": "26379"
},
{
"host": "XXX.XX.X.XX",
"port": "26379"
},
{
"host": "XXX.XX.X.XX",
"port": "26379"
}
];
var SENTINEL_ENDPOINTS_PROD = [
{
"host": "XXX.XX.X.XX",
"port": "26379"
},
{
"host": "XXX.XX.X.XX",
"port": "26379"
},
{
"host": "XXX.X.XX.XX",
"port": "26379"
}
];
var clientStaging = new Redis({
sentinels: SENTINEL_ENDPOINTS_STAGING,
name: REDIS_SENTINEL_MASTER_NAME_STAGING,
db: 0
});
var clientProd = new Redis({
sentinels: SENTINEL_ENDPOINTS_PROD,
name: REDIS_SENTINEL_MASTER_NAME_PROD,
db: 0
});
var dbStaging = {};
var dbProd = {};
async.parallel([
function (callback) {
clientStaging.keys('*', function (err, keys) {
async.each(keys, function (key, cb) {
clientStaging.get(key, function (err, result) {
//console.log('staging> key: "'+key+'", result:', result);
dbStaging[key] = result;
cb(err);
});
}, function done(err, res) {
callback(err, res);
});
});
},
function (callback) {
clientProd.keys('*', function (err, keys) {
async.each(keys, function (key, cb) {
clientProd.get(key, function (err, result) {
//console.log('production> key: "'+key+'", result:', result);
dbProd[key] = result;
cb(err);
});
}, function done(err, res) {
callback(err, res);
});
});
}
],
function done(err, results) {
if (err) {
throw err;
}
else {
diffDbs();
}
});
function diffDbs() {
Object.keys(dbStaging).forEach(function (key) {
if (!dbProd.hasOwnProperty(key)) {
console.log('staging redis has key, prod redis does not:',key);
}
else {
var stagingVal = dbStaging[key];
var prodVal = dbProd[key];
if(String(stagingVal).valueOf() != String(prodVal).valueOf()){
console.log('staging redis and prod redis have DIFFERENT values for key:',key);
}
}
});
Object.keys(dbProd).forEach(function (key) {
if (!dbStaging.hasOwnProperty(key)) {
console.log('prod redis has key, staging redis does not:',key);
}
else {
var stagingVal = String(dbStaging[key]).replace(/ /g, ''); //remove all whitespace
var prodVal = String(dbProd[key]).replace(/ /g, ''); //remove all whitespace
if(stagingVal.valueOf() != prodVal.valueOf()){
console.log('staging redis and prod redis have DIFFERENT values for key:',key);
}
}
});
process.exit(0);
}