Skip to content
Snippets Groups Projects
Commit 7c5a9190 authored by Andrew Newdigate's avatar Andrew Newdigate
Browse files

✨ Feature: write stats to Kibana

parent d5b29fbf
No related branches found
No related tags found
1 merge request!6✨ Feature: write stats to Kibana
Hide whitespace changes
Inline Side-by-side
lib/env.js
+ 2
− 3
View file @ 7c5a9190
@@ -4,8 +4,8 @@
exports.create = function(configDirectory) {
var config = require('./config').create(configDirectory);
 
var stats = require('./stats').create({ config: config });
var logger = require('./logger').create({ config: config, stats: stats });
var logger = require('./logger').create({ config: config });
var stats = require('./stats').create({ config: config, logger: logger });
var errorReporter = require('./error-reporter').create({ config: config, logger: logger });
var redisClient = require('./redis-client');
var errorHandlerMiddleware = require('./middlewares/error-handler');
@@ -154,6 +154,5 @@ exports.create = function(configDirectory) {
}
});
 
return env;
};
lib/logger.js
+ 83
− 157
View file @ 7c5a9190
/*jshint globalstrict:true, trailing:false, unused:true, node:true */
"use strict";
 
var processName = require('./process-name')
exports.create = function(options) {
function createWinstonContainer(config) {
var winston = require("winston");
var fs = require('fs');
var path = require('path');
var transformMeta = require('./transform-meta');
var nconf = options.config;
if(!nconf) throw new Error("options must contain config");
var defaultLogger = new winston.Logger({
transports: []
});
function statFile(fileTransport) {
if(!fileTransport) return;
var fullname = path.join(fileTransport.dirname, fileTransport._getFile(false));
function reopen() {
fileTransport.close();
fileTransport._stream = null;
fileTransport.once('open', function() {
defaultLogger.info('Log rotation completed');
console.log('Log rotation completed');
});
}
fs.stat(fullname, function (err, stat) {
if (err && err.code === 'ENOENT') {
console.log('Log file no longer exists. Reopening');
return reopen();
}
if(fileTransport._stream && fileTransport._stream.fd) {
fs.fstat(fileTransport._stream.fd, function(err2, fstat) {
if(stat.dev !== fstat.dev || stat.ino !== fstat.ino) {
console.log('File inode mismatch. Reopening');
return reopen();
}
});
}
});
}
var periodicListenerConfigured = false;
function periodicallyStatFile() {
if(periodicListenerConfigured) return;
periodicListenerConfigured = true;
setInterval(function() {
statFile(defaultLogger.transports.file);
}, 30000);
}
var hupListenerConfigured = false;
function reopenTransportOnHupSignal() {
if(hupListenerConfigured) return;
hupListenerConfigured = true;
process.on('SIGHUP', function() {
console.log('Caught SIGHUP, attempting logfile rotation');
defaultLogger.info('Caught SIGHUP, attempting logfile rotation');
statFile(defaultLogger.transports.file);
});
}
function attachEventHandlers() {
var stats = options.stats;
if(stats) {
defaultLogger.on('logging', function (transport, level) {
try {
switch(level) {
case 'warn':
stats.eventHF('logged.warn');
break;
case 'error':
stats.eventHF('logged.error');
break;
case 'fatal':
stats.eventHF('logged.fatal');
break;
}
} catch(e) {
console.log('Error calling stats service', e);
}
});
}
defaultLogger.on('error', function (err) {
console.error('Logging error: ' + err, err);
if(err && err.stack) {
console.error(err.stack);
}
});
var processName = require('./process-name')
 
}
function getTransports() {
var results = [];
 
function configureTransports() {
for (var name in defaultLogger.transports) {
defaultLogger.remove({ name: name });
}
if(nconf.get("logging:logToUDP")) {
if(config.get("logging:logToUDP")) {
var appName = processName.getGenericProcessName();
if (appName) {
require('winston-udp')
var os = require('os')
defaultLogger.add(winston.transports.UDP, {
host: nconf.get('logging:udpHost') || '127.0.0.1',
port: nconf.get('logging:udpPort') || 24224,
var udpTransport = new winston.transports.UDP({
host: config.get("logging:udpHost") || '127.0.0.1',
port: config.get("logging:udpPort") || 24224,
level: config.get("logging:level"),
timestamp: function() {
return Date.now();
},
@@ -132,70 +32,96 @@ exports.create = function(options) {
});
}
});
results.push(udpTransport);
}
}
if (nconf.get('logging:logToFile') && nconf.get('LOG_FILE')) {
defaultLogger.add(winston.transports.File, {
filename: nconf.get('LOG_FILE'),
level: nconf.get("logging:level"),
timestamp: true,
maxFiles: null,
maxsize: null,
json: false
if(!config.get("logging:disableConsole") && !process.env.DISABLE_CONSOLE_LOGGING) {
var consoleTransport = new winston.transports.Console({
colorize: config.get("logging:colorize"),
timestamp: config.get("logging:timestamp"),
level: config.get("logging:level"),
prettyPrint: config.get("logging:prettyPrint")
});
 
var fileTransport = defaultLogger.transports.file;
periodicallyStatFile(fileTransport);
reopenTransportOnHupSignal(fileTransport);
results.push(consoleTransport);
}
 
} else {
if(!nconf.get("logging:disableConsole") && !process.env.DISABLE_CONSOLE_LOGGING) {
return results;
}
 
defaultLogger.add(winston.transports.Console, {
colorize: nconf.get("logging:colorize"),
timestamp: nconf.get("logging:timestamp"),
level: nconf.get("logging:level"),
prettyPrint: nconf.get("logging:prettyPrint")
});
var container = new winston.Container({
transports: getTransports()
});
 
var consoleTransport = defaultLogger.transports.console;
attachEventHandlers(consoleTransport);
}
}
return container;
}
 
}
exports.create = function(options) {
var transformMeta = require('./transform-meta');
 
configureTransports();
var config = options.config;
if(!config) throw new Error("options must contain config");
 
var logLevel = nconf.get("logging:level");
var container = createWinstonContainer(config);
var statsdClient = require('./stats-client').create({
config: config,
prefix: config.get('stats:statsd:prefix')
});
 
nconf.events.on('reload', function() {
if(logLevel === nconf.get("logging:level")) {
return;
var loggers = {};
function get(category) {
if (loggers[category]) {
return loggers[category];
}
 
logLevel = nconf.get("logging:level");
console.log("Reconfiguring log transports");
// Create and setup a logger
var logger = container.get(category);
loggers[category] = logger;
 
configureTransports();
});
var logTags;
if (category) {
logTags = ['category:' + category];
} else {
logTags = undefined;
}
logger.on('logging', function (transport, level) {
switch(level) {
case 'warn':
statsdClient.increment('logged.warn', 1, 0.1, logTags);
break;
case 'error':
statsdClient.increment('logged.error', 1, 0.1, logTags);
break;
case 'fatal':
statsdClient.increment('logged.fatal', 1, 0.1, logTags);
break;
}
});
 
Error.stackTraceLimit = 100;
 
logger.on('error', function (err) {
console.error('Logging error: ' + err, err);
if(err && err.stack) {
console.error(err.stack);
}
});
 
logger.rewriters.push(function(level, msg, meta) {
meta = transformMeta(meta);
if (category) {
meta.category = category;
}
return meta;
});
 
function customLogger(logger) {
return ['silly', 'verbose', 'info', 'debug', 'warn', 'error', 'fatal'].reduce(function(memo, method) {
memo[method] = function(message, meta) {
if(arguments.length === 1) {
logger[method](message);
} else {
logger[method](message, transformMeta(meta));
}
};
return memo;
}, {});
return logger;
}
 
return customLogger(defaultLogger);
var defaultLogger = get('');
defaultLogger.get = get;
return defaultLogger;
};
lib/stats.js
+ 23
− 29
View file @ 7c5a9190
/*jshint globalstrict:true, trailing:false, unused:true, node:true */
"use strict";
 
var _ = require('lodash');
function emergencyLog() {
console.error.apply(console, arguments);
}
@@ -10,25 +11,18 @@ function getBucketFromUserId(userId) {
return (parseInt(lastChar+'', 16) % 2) ? 'A' : 'B';
}
 
function configureConsole(statsHandlers) {
statsHandlers.event.push(function(eventName) {
console.log('stat: event: ' + eventName);
});
statsHandlers.charge.push(function (userId, usd) {
console.log('stat: charge:', userId, usd);
});
statsHandlers.eventHF.push(function(eventName, count, frequency, tags) {
console.log('stat: eventHF: ' + eventName + ' +' + count + (tags ? ' ' + tags.join(', ') : ''));
});
function configureLogger(statsHandlers, config, logger) {
statsHandlers.event.push(function(eventName, properties) {
if (!properties) return;
 
statsHandlers.gaugeHF.push(function(gaugeName, value, frequency, tags) {
console.log('stat: gaugeHF: ' + gaugeName + ": " + value + (tags ? ' ' + tags.join(', ') : ''));
});
// Strip out dodgy named attributes which may screw with Kibana
var meta = _.omit(properties, 'agent:type', 'agent:family', 'agent:version',
'agent:device:family', 'agent:os:family',
'agent:os:version');
meta.eventName = eventName;
 
statsHandlers.responseTime.push(function(timerName, duration, tags) {
console.log('stat: responseTime: ' + timerName + ": " + duration + (tags ? ' ' + tags.join(', ') : ''));
logger.info('stats.event', meta);
});
}
 
@@ -206,19 +200,18 @@ function configureIntercom(statsHandlers, config) {
 
statsHandlers.userUpdate.push(function(user/*, properties*/) {
var created_at = new Date(user._id.getTimestamp());
//var now = Number(Date.now()/1000).toFixed(0);
 
var profile = {
"email" : user.email,
"user_id" : user._id,
"name" : user.displayName,
"created_at" : created_at,
//"last_impression_at" : now,
"email": user.email,
"user_id": user._id,
"name": user.displayName,
"created_at": created_at,
//"last_impression_at": now,
"update_last_request_at": true,
"custom_data": {
"username" : user.username,
"username": user.username,
"bucket": getBucketFromUserId(user._id),
"state" : user.state || 'ACTIVE',
"state": user.state || 'ACTIVE',
}
};
 
@@ -360,6 +353,7 @@ exports.create = function(options) {
// we'll forego logging in the stats module rather than the
// other way around
var config = options.config;
var logger = options.logger;
 
var statsHandlers = {
trackRequest: [],
@@ -376,14 +370,14 @@ exports.create = function(options) {
var gaEnabled = config.get('stats:ga:enabled');
var statsdEnabled = config.get("stats:statsd:enabled");
var cubeEnabled = config.get("stats:cube:enabled");
var consoleEnabled = config.get("stats:console:enabled");
var loggerEnabled = config.get("stats:logger:enabled");
var intercomEnabled = config.get("stats:intercom:enabled");
 
/**
* console
*/
if (consoleEnabled) {
configureConsole(statsHandlers, config);
if (loggerEnabled) {
configureLogger(statsHandlers, config, logger);
}
 
/**
package.json
+ 2
− 2
View file @ 7c5a9190
@@ -48,9 +48,9 @@
"winston-udp": "^0.0.6"
},
"devDependencies": {
"eslint": "^2.11.1",
"eslint": "^2.13.1",
"eslint-plugin-mocha": "^3.0.0",
"eslint-plugin-node": "^1.4.0",
"eslint-plugin-node": "^2.0.0",
"mocha": "^1.18.2",
"retire": "^0.2.1",
"temp": "^0.8.3"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment