NestJS course Β· Module 6: Error Handling and Monitoring
PROJECT: Complete Legion Monitoring System
In this lesson4
Chief technology legate! Consul Caesar.js has entrusted you with the ultimate challenge - creating a complete monitoring system for the entire Roman legion. This will be a real test of all the skills acquired in the Error Handling & Monitoring module!
Project Specification
Your system must be a complete solution for monitoring a legion consisting of many Roman forts. Each fort is a separate NestJS application instance, and you are building the central monitoring system.
Functional Requirements
1. Multi-Legion Monitoring
- Central Dashboard - overview of all forts in the legion
- Legion Registration - automatic registration of new forts
- Health Checks - health monitoring of each fort
- Legion Analytics - legion-wide analytics
2. Real-time Error Tracking
- Error Aggregation - collecting errors from all forts
- Error Classification - categorizing errors by type and severity
- Error Correlation - linking similar errors
- Alert System - intelligent notifications
3. Performance Monitoring
- System Metrics - CPU, memory, disk, network
- Application Metrics - response time, throughput, error rate
- Business Metrics - custom metrics specific to the legion
- SLA Monitoring - tracking performance targets
4. Advanced Alerting
- Smart Alerts - machine learning-based alerts
- Escalation Policies - alert escalation rules
- Multi-channel Notifications - Slack, email, SMS, webhook
- Alert Correlation - grouping related alerts
System Architecture
// Architecture Overview
/*
βββββββββββββββββββ βββββββββββββββββββ βββββββββββββββββββ
β Legion Alpha β β Legion Beta β β Legion Gamma β
β (NestJS) β β (NestJS) β β (NestJS) β
β β β β β β
β - Error Agent β β - Error Agent β β - Error Agent β
β - Metrics Agent β β - Metrics Agent β β - Metrics Agent β
β - Health Check β β - Health Check β β - Health Check β
βββββββββββ¬ββββββββ βββββββββββ¬ββββββββ βββββββββββ¬ββββββββ
β β β
ββββββββββββββββββββββββΌβββββββββββββββββββββββ
β
βββββββββββββββ΄ββββββββββββββ
β Legion Monitor Center β
β (Main NestJS App) β
β β
β - Data Aggregation β
β - Alert Management β
β - Dashboard API β
β - Analytics Engine β
β - Notification System β
βββββββββββββββ¬ββββββββββββββ
β
βββββββββββββββ΄ββββββββββββββ
β External Systems β
β β
β - Prometheus/Grafana β
β - Elasticsearch/Kibana β
β - Slack/Teams β
β - PagerDuty β
βββββββββββββββββββββββββββββ
*/System Implementation
1. Legion Agent (for each fort)
// agents/cohort-agent.service.ts
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { HttpService } from '@nestjs/axios';
import { ConfigService } from '@nestjs/config';
export interface LegionMetrics {
cohortId: string;
cohortName: string;
centurion: string;
timestamp: Date;
system: {
cpu: number;
memory: number;
disk: number;
network: {
bytesIn: number;
bytesOut: number;
};
};
application: {
uptime: number;
activeConnections: number;
requestsPerSecond: number;
errorRate: number;
responseTime: {
avg: number;
p95: number;
p99: number;
};
};
business: {
tributesFound: number;
legionActivity: number;
missionsCompleted: number;
battlesWon: number;
};
health: {
overall: 'healthy' | 'warning' | 'critical';
services: Record<string, 'up' | 'down' | 'degraded'>;
};
}
@Injectable()
export class LegionAgentService implements OnModuleInit {
private readonly logger = new Logger(LegionAgentService.name);
private readonly cohortId: string;
private readonly legionCenterUrl: string;
private isRegistered = false;
constructor(
private readonly httpService: HttpService,
private readonly configService: ConfigService,
) {
this.cohortId = this.configService.get('COHORT_ID') || this.generateLegionId();
this.legionCenterUrl = this.configService.get('LEGION_CENTER_URL');
}
async onModuleInit() {
await this.registerWithLegion();
}
private async registerWithLegion() {
try {
const registrationData = {
cohortId: this.cohortId,
cohortName: this.configService.get('COHORT_NAME', 'Unknown Vessel'),
centurion: this.configService.get('CENTURION_NAME', 'Anonymous Centurion'),
version: process.env.npm_package_version || '1.0.0',
startTime: new Date(),
location: {
latitude: parseFloat(this.configService.get('COHORT_LAT', '0')),
longitude: parseFloat(this.configService.get('COHORT_LNG', '0')),
},
};
await this.httpService.post(
`${this.legionCenterUrl}/legion/cohorts/register`,
registrationData
).toPromise();
this.isRegistered = true;
this.logger.log(` Legion ${this.cohortId} registered with legion center`);
} catch (error) {
this.logger.error('Failed to register with legion center:', error.message);
}
}
@Cron('0 * * * * *') // Every minute
async sendMetricsToLegion() {
if (!this.isRegistered) {
await this.registerWithLegion();
return;
}
try {
const metrics = await this.collectLegionMetrics();
await this.httpService.post(
`${this.legionCenterUrl}/legion/metrics`,
metrics
).toPromise();
this.logger.debug(` Metrics sent to legion center`);
} catch (error) {
this.logger.error('Failed to send metrics to legion center:', error.message);
}
}
async collectLegionMetrics(): Promise<LegionMetrics> {
const memUsage = process.memoryUsage();
const uptime = process.uptime();
return {
cohortId: this.cohortId,
cohortName: this.configService.get('COHORT_NAME', 'Unknown Vessel'),
centurion: this.configService.get('CENTURION_NAME', 'Anonymous Centurion'),
timestamp: new Date(),
system: {
cpu: await this.getCpuUsage(),
memory: Math.round((memUsage.heapUsed / memUsage.heapTotal) * 100),
disk: await this.getDiskUsage(),
network: await this.getNetworkStats(),
},
application: {
uptime,
activeConnections: await this.getActiveConnections(),
requestsPerSecond: await this.getRequestsPerSecond(),
errorRate: await this.getErrorRate(),
responseTime: await this.getResponseTimeStats(),
},
business: {
tributesFound: await this.getTributesFound(),
legionActivity: await this.getLegionActivity(),
missionsCompleted: await this.getMissionsCompleted(),
battlesWon: await this.getBattlesWon(),
},
health: await this.getHealthStatus(),
};
}
private async getCpuUsage(): Promise<number> {
return new Promise((resolve) => {
const startUsage = process.cpuUsage();
setTimeout(() => {
const endUsage = process.cpuUsage(startUsage);
const totalUsage = endUsage.user + endUsage.system;
const percentage = (totalUsage / 1000000) * 100;
resolve(Math.min(100, percentage));
}, 100);
});
}
private async getDiskUsage(): Promise<number> {
// Simplified disk usage - in real implementation use fs.statvfs
return Math.random() * 100;
}
private async getNetworkStats() {
// Simplified network stats - in real implementation read from /proc/net/dev
return {
bytesIn: Math.floor(Math.random() * 1000000),
bytesOut: Math.floor(Math.random() * 1000000),
};
}
private async getActiveConnections(): Promise<number> {
// Implementation depends on your server setup
return Math.floor(Math.random() * 100);
}
private async getRequestsPerSecond(): Promise<number> {
// Implementation using metrics collected by interceptors
return Math.random() * 1000;
}
private async getErrorRate(): Promise<number> {
// Implementation using error tracking
return Math.random() * 10;
}
private async getResponseTimeStats() {
return {
avg: 50 + Math.random() * 200,
p95: 100 + Math.random() * 400,
p99: 200 + Math.random() * 800,
};
}
private async getTributesFound(): Promise<number> {
// Business metric - implement based on your tribute service
return Math.floor(Math.random() * 50);
}
private async getLegionActivity(): Promise<number> {
// Business metric - implement based on your legion service
return Math.floor(Math.random() * 100);
}
private async getMissionsCompleted(): Promise<number> {
// Business metric - implement based on your mission service
return Math.floor(Math.random() * 20);
}
private async getBattlesWon(): Promise<number> {
// Business metric - implement based on your battle service
return Math.floor(Math.random() * 10);
}
private async getHealthStatus() {
// Implement comprehensive health checks
const services = {
database: Math.random() > 0.1 ? 'up' : 'down',
redis: Math.random() > 0.05 ? 'up' : 'down',
api: Math.random() > 0.02 ? 'up' : 'degraded',
auth: Math.random() > 0.01 ? 'up' : 'down',
};
const downServices = Object.values(services).filter(s => s === 'down').length;
const degradedServices = Object.values(services).filter(s => s === 'degraded').length;
let overall: 'healthy' | 'warning' | 'critical' = 'healthy';
if (downServices > 0) {
overall = 'critical';
} else if (degradedServices > 0) {
overall = 'warning';
}
return { overall, services };
}
private generateLegionId(): string {
return `cohort-${Math.random().toString(36).substr(2, 9)}`;
}
async sendErrorToLegion(error: any, context?: any) {
if (!this.isRegistered) return;
try {
const errorData = {
cohortId: this.cohortId,
timestamp: new Date(),
error: {
message: error.message,
stack: error.stack,
type: error.constructor.name,
code: error.code,
},
context: {
url: context?.url,
method: context?.method,
user: context?.user?.id,
requestId: context?.requestId,
...context,
},
severity: this.classifyErrorSeverity(error),
};
await this.httpService.post(
`${this.legionCenterUrl}/legion/errors`,
errorData
).toPromise();
this.logger.debug(' Error sent to legion center');
} catch (err) {
this.logger.error('Failed to send error to legion center:', err.message);
}
}
private classifyErrorSeverity(error: any): 'low' | 'medium' | 'high' | 'critical' {
if (error.name?.includes('Database') || error.code === 'ECONNREFUSED') {
return 'critical';
}
if (error.status >= 500) {
return 'high';
}
if (error.status >= 400) {
return 'medium';
}
return 'low';
}
}2. Legion Command - Central Monitoring Hub
// legion-center/legion-monitor.service.ts
import { Injectable, Logger } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { Cron } from '@nestjs/schedule';
export interface Legion {
cohorts: Map<string, LegionInfo>;
totalLegions: number;
healthyLegions: number;
warningLegions: number;
criticalLegions: number;
lastUpdate: Date;
}
export interface LegionInfo {
cohortId: string;
cohortName: string;
centurion: string;
lastSeen: Date;
metrics: LegionMetrics;
status: 'healthy' | 'warning' | 'critical' | 'offline';
alerts: Alert[];
errors: ErrorEvent[];
}
@Injectable()
export class LegionMonitorService {
private readonly logger = new Logger(LegionMonitorService.name);
private legion: Legion = {
cohorts: new Map(),
totalLegions: 0,
healthyLegions: 0,
warningLegions: 0,
criticalLegions: 0,
lastUpdate: new Date(),
};
constructor(private readonly eventEmitter: EventEmitter2) {}
registerLegion(registrationData: any) {
const cohortInfo: LegionInfo = {
cohortId: registrationData.cohortId,
cohortName: registrationData.cohortName,
centurion: registrationData.centurion,
lastSeen: new Date(),
metrics: null,
status: 'healthy',
alerts: [],
errors: [],
};
this.legion.cohorts.set(registrationData.cohortId, cohortInfo);
this.updateLegionStats();
this.logger.log(` Legion registered: ${cohortInfo.cohortName} (${cohortInfo.cohortId})`);
this.eventEmitter.emit('cohort.registered', cohortInfo);
}
updateLegionMetrics(cohortId: string, metrics: LegionMetrics) {
const cohort = this.legion.cohorts.get(cohortId);
if (!cohort) {
this.logger.warn(`Received metrics from unregistered cohort: ${cohortId}`);
return;
}
cohort.metrics = metrics;
cohort.lastSeen = new Date();
cohort.status = this.determineLegionStatus(cohort);
this.updateLegionStats();
this.analyzeMetricsForAlerts(cohort);
this.eventEmitter.emit('cohort.metrics.updated', { cohortId, metrics });
}
recordLegionError(cohortId: string, errorData: any) {
const cohort = this.legion.cohorts.get(cohortId);
if (!cohort) {
this.logger.warn(`Received error from unregistered cohort: ${cohortId}`);
return;
}
const errorEvent: ErrorEvent = {
id: `error-${Date.now()}-${Math.random()}`,
timestamp: new Date(),
cohortId,
...errorData,
};
cohort.errors.push(errorEvent);
// Keep only last 100 errors per cohort
if (cohort.errors.length > 100) {
cohort.errors = cohort.errors.slice(-100);
}
this.processError(errorEvent);
this.eventEmitter.emit('cohort.error', errorEvent);
}
private determineLegionStatus(cohort: LegionInfo): 'healthy' | 'warning' | 'critical' | 'offline' {
if (!cohort.metrics) return 'offline';
const timeSinceLastSeen = Date.now() - cohort.lastSeen.getTime();
if (timeSinceLastSeen > 300000) { // 5 minutes
return 'offline';
}
if (cohort.metrics.health.overall === 'critical') {
return 'critical';
}
const { cpu, memory } = cohort.metrics.system;
const { errorRate } = cohort.metrics.application;
if (cpu > 90 || memory > 90 || errorRate > 10) {
return 'critical';
}
if (cpu > 70 || memory > 70 || errorRate > 5 || cohort.metrics.health.overall === 'warning') {
return 'warning';
}
return 'healthy';
}
private updateLegionStats() {
this.legion.totalLegions = this.legion.cohorts.size;
this.legion.healthyLegions = 0;
this.legion.warningLegions = 0;
this.legion.criticalLegions = 0;
for (const cohort of this.legion.cohorts.values()) {
switch (cohort.status) {
case 'healthy':
this.legion.healthyLegions++;
break;
case 'warning':
this.legion.warningLegions++;
break;
case 'critical':
this.legion.criticalLegions++;
break;
}
}
this.legion.lastUpdate = new Date();
}
private analyzeMetricsForAlerts(cohort: LegionInfo) {
const alerts: Alert[] = [];
// CPU Alert
if (cohort.metrics.system.cpu > 90) {
alerts.push(this.createAlert(
'high-cpu',
'critical',
`High CPU usage on ${cohort.cohortName}`,
`CPU usage is ${cohort.metrics.system.cpu.toFixed(1)}%`,
cohort.cohortId
));
}
// Memory Alert
if (cohort.metrics.system.memory > 90) {
alerts.push(this.createAlert(
'high-memory',
'critical',
`High memory usage on ${cohort.cohortName}`,
`Memory usage is ${cohort.metrics.system.memory.toFixed(1)}%`,
cohort.cohortId
));
}
// Error Rate Alert
if (cohort.metrics.application.errorRate > 10) {
alerts.push(this.createAlert(
'high-error-rate',
'high',
`High error rate on ${cohort.cohortName}`,
`Error rate is ${cohort.metrics.application.errorRate.toFixed(2)}%`,
cohort.cohortId
));
}
// Response Time Alert
if (cohort.metrics.application.responseTime.p99 > 5000) {
alerts.push(this.createAlert(
'slow-response',
'medium',
`Slow response times on ${cohort.cohortName}`,
`P99 response time is ${cohort.metrics.application.responseTime.p99.toFixed(0)}ms`,
cohort.cohortId
));
}
// Business Metrics Alerts
if (cohort.metrics.business.tributesFound === 0) {
alerts.push(this.createAlert(
'no-tributes',
'low',
`No tributes found on ${cohort.cohortName}`,
'Legion legion might need motivation!',
cohort.cohortId
));
}
// Add new alerts and trigger them
for (const alert of alerts) {
if (!cohort.alerts.some(a => a.type === alert.type && a.severity === alert.severity)) {
cohort.alerts.push(alert);
this.eventEmitter.emit('alert.triggered', alert);
}
}
}
@Cron('0 */5 * * * *') // Every 5 minutes
checkLegionHealth() {
const offlineLegions = [];
const now = Date.now();
for (const [cohortId, cohort] of this.legion.cohorts) {
const timeSinceLastSeen = now - cohort.lastSeen.getTime();
if (timeSinceLastSeen > 300000) { // 5 minutes offline
cohort.status = 'offline';
offlineLegions.push(cohort);
}
}
if (offlineLegions.length > 0) {
for (const cohort of offlineLegions) {
const alert = this.createAlert(
'cohort-offline',
'critical',
`Legion ${cohort.cohortName} is offline`,
`No communication for ${Math.round((now - cohort.lastSeen.getTime()) / 60000)} minutes`,
cohort.cohortId
);
this.eventEmitter.emit('alert.triggered', alert);
}
}
this.updateLegionStats();
}
getLegionOverview(): Legion {
return this.legion;
}
getLegionDetails(cohortId: string): LegionInfo | null {
return this.legion.cohorts.get(cohortId) || null;
}
getLegionMetrics() {
const cohorts = Array.from(this.legion.cohorts.values());
return {
legion: this.legion,
aggregatedMetrics: {
totalCpu: cohorts.reduce((sum, cohort) => sum + (cohort.metrics?.system.cpu || 0), 0),
totalMemory: cohorts.reduce((sum, cohort) => sum + (cohort.metrics?.system.memory || 0), 0),
totalErrors: cohorts.reduce((sum, cohort) => sum + cohort.errors.length, 0),
totalTributes: cohorts.reduce((sum, cohort) => sum + (cohort.metrics?.business.tributesFound || 0), 0),
averageResponseTime: cohorts.reduce((sum, cohort) => sum + (cohort.metrics?.application.responseTime.avg || 0), 0) / cohorts.length,
},
topPerformers: this.getTopPerformingLegions(),
worstPerformers: this.getWorstPerformingLegions(),
};
}
private getTopPerformingLegions(): LegionInfo[] {
return Array.from(this.legion.cohorts.values())
.filter(cohort => cohort.metrics && cohort.status === 'healthy')
.sort((a, b) => b.metrics.business.tributesFound - a.metrics.business.tributesFound)
.slice(0, 5);
}
private getWorstPerformingLegions(): LegionInfo[] {
return Array.from(this.legion.cohorts.values())
.filter(cohort => cohort.metrics && cohort.status !== 'healthy')
.sort((a, b) => b.metrics.application.errorRate - a.metrics.application.errorRate)
.slice(0, 5);
}
private createAlert(type: string, severity: string, title: string, description: string, cohortId: string): Alert {
return {
id: `alert-${Date.now()}-${Math.random()}`,
type,
severity,
title,
description,
cohortId,
timestamp: new Date(),
resolved: false,
};
}
private processError(errorEvent: ErrorEvent) {
// Implement error processing logic
// - Error correlation
// - Pattern detection
// - Automatic alert generation
this.logger.debug(`Processing error from ${errorEvent.cohortId}: ${errorEvent.error.message}`);
}
}Tasks to Complete
Part 1: Basic System (4-6 hours)
- Legion Agent Implementation
- Implement LegionAgentService with real metrics
- Add error tracking interceptor
- Create health check endpoints
- Add automatic registration with Legion Command
- Legion Command Core
- Implement LegionMonitorService
- Create API endpoints for registration and metrics
- Add basic dashboard endpoint
- Implement real-time WebSocket communication
Part 2: Advanced Features (6-8 hours)
- Intelligent Alerting
- Machine learning for problem prediction
- Alert correlation and deduplication
- Escalation policies
- Multi-channel notifications (Slack, email, SMS)
- Analytics Engine
- Trend analysis i forecasting
- Anomaly detection
- Performance baselines
- Business metrics correlation
Part 3: Enterprise Features (8-10 hours)
- Advanced Monitoring
- Custom metrics framework
- Distributed tracing
- Log aggregation
- SLA monitoring
- Integration & Deployment
- Prometheus/Grafana integration
- Docker containerization
- Kubernetes deployment
- CI/CD pipeline
Part 4: Testing & Documentation (4-6 hours)
- Comprehensive Testing
- Unit tests for all components
- Integration tests for legion communication
- Load testing for scalability
- E2E testing of the complete workflow
- Documentation & Deployment Guide
- API documentation (Swagger)
- Deployment guide
- Monitoring runbooks
- Troubleshooting guide
Evaluation Criteria
Excellency Criteria (90-100%)
Complete implementation of all required functionalities Performance - system handles >100 forts simultaneously Scalability - easy addition of new metric and alert types Reliability - system runs stably for 24h without interruptions Security - proper authentication and authorization Code Quality - clean code, proper patterns, comprehensive tests Documentation - complete documentation and deployment guides
Advanced Features (Bonus Points)
Machine Learning - predictive alerting and anomaly detection Auto-scaling - automatic resource adjustment based on metrics Multi-region - support for geographically distributed legions Mobile App - React Native app for legion monitoring Voice Alerts - integration with Amazon Alexa for voice notifications
Good luck, legate! Your legion is counting on you - create a monitoring system that will be a legend across the provinces of technology!
Code for this lesson: src/monitoring/legion-monitoring-project.ts
// PROJECT: Complete Legion Monitoring System
// Combining Exception Filters, Logging, Health Checks and Alerts
import { Injectable, Logger } from '@nestjs/common';
import { Controller, Get } from '@nestjs/common';
// ===========================================
// 1. Central monitoring system
// ===========================================
@Injectable()
export class LegionMonitoringService {
private logger = new Logger('LegionMonitor');
// Registry of cohorts in the legion
private cohorts = new Map<string, {
name: string;
status: 'operational' | 'warning' | 'critical' | 'offline';
lastHealthCheck: Date;
metrics: { cpu: number; memory: number; errors: number };
}>();
// Cohort registration
registerCohort(id: string, name: string) {
this.cohorts.set(id, {
name,
status: 'operational',
lastHealthCheck: new Date(),
metrics: { cpu: 0, memory: 0, errors: 0 },
});
this.logger.log('Registered cohort: ' + name);
}
// Updating metrics
updateMetrics(id: string, metrics: { cpu: number; memory: number; errors: number }) {
const cohort = this.cohorts.get(id);
if (!cohort) return;
cohort.metrics = metrics;
cohort.lastHealthCheck = new Date();
// Automatic status change based on metrics
if (metrics.errors > 10 || metrics.cpu > 95) {
cohort.status = 'critical';
this.logger.error('CRITICAL: ' + cohort.name);
} else if (metrics.errors > 5 || metrics.cpu > 80) {
cohort.status = 'warning';
this.logger.warn('WARNING: ' + cohort.name);
} else {
cohort.status = 'operational';
}
}
// Entire legion dashboard
getLegionDashboard() {
const cohorts = Array.from(this.cohorts.entries()).map(([id, data]) => ({
id, ...data,
}));
const operational = cohorts.filter(s => s.status === 'operational').length;
const warnings = cohorts.filter(s => s.status === 'warning').length;
const critical = cohorts.filter(s => s.status === 'critical').length;
return {
totalCohorts: cohorts.length,
operational,
warnings,
critical,
legionHealth: critical > 0 ? 'ALERT' : warnings > 0 ? 'CAUTION' : 'OK',
cohorts,
generatedAt: new Date().toISOString(),
};
}
}
// ===========================================
// 2. Dashboard Controller
// ===========================================
@Controller('legion-monitor')
export class LegionMonitorController {
constructor(private monitorService: LegionMonitoringService) {}
@Get('dashboard')
getDashboard() {
return this.monitorService.getLegionDashboard();
}
}
// ===========================================
// 3. Simulation
// ===========================================
const monitor = new LegionMonitoringService();
monitor.registerCohort('S1', 'Imperator');
monitor.registerCohort('S2', 'Aquila');
monitor.registerCohort('S3', 'Victoria');
monitor.updateMetrics('S1', { cpu: 45, memory: 60, errors: 0 });
monitor.updateMetrics('S2', { cpu: 85, memory: 70, errors: 7 });
monitor.updateMetrics('S3', { cpu: 20, memory: 30, errors: 0 });
const dashboard = monitor.getLegionDashboard();
console.log('=== Legion Dashboard ===');
console.log('Cohorts:', dashboard.totalCohorts);
console.log('Operational:', dashboard.operational);
console.log('Warnings:', dashboard.warnings);
console.log('Critical:', dashboard.critical);
console.log('Legion status:', dashboard.legionHealth);
Docs entries used in this lesson
Spotted a mistake in this lesson?