From fa941a12728c9e7f6c55d8991ea54e43c8efb1b6 Mon Sep 17 00:00:00 2001 From: Martin <42486612+kriiv@users.noreply.github.com> Date: Sun, 22 Sep 2024 15:57:24 +1000 Subject: [PATCH 1/2] added logging, improved readme --- package.json | 4 +++- readme.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++--- server.js | 51 +++++++++++++++++++++++++++++++-------------- 3 files changed, 94 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index bf3ff08..19d95d8 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "better-queue": "^3.8.12", "dotenv": "^16.4.5", "mailparser": "^3.7.1", - "smtp-server": "^3.13.5" + "smtp-server": "^3.13.5", + "winston": "^3.x.x", + "winston-daily-rotate-file": "^4.x.x" } } diff --git a/readme.md b/readme.md index a47a5bd..2bfd463 100644 --- a/readme.md +++ b/readme.md @@ -72,11 +72,11 @@ Using inbound parse for something interesting? Please let me know, I'd love to h ## Todo - Rate limiting -- Log Storage +- ~~Log Storage~~ (completed) ## Contributing -Contributions are welcome! Please feel free to submit a Pull Request. +Contributions are welcome! Please feel free to submit a Pull Request or get in touch. ## License @@ -84,4 +84,57 @@ This project is licensed under the MIT License. This means you are free to use, ## Disclaimer -Please ensure you have the necessary permissions and security measures in place when deploying an SMTP server. Depending on your firewall configuration, you may be exposing this service to the internet. \ No newline at end of file +Please ensure you have the necessary permissions and security measures in place when deploying an SMTP server. Depending on your firewall configuration, you may be exposing this service to the internet. + +## Security Considerations + +When deploying this SMTP server, please keep the following security considerations in mind: + +- Ensure that your server is properly secured and that only authorized IPs can access the SMTP port. +- Use strong, unique passwords for your AWS credentials and keep them secure. +- Regularly update the Node.js runtime and all dependencies to their latest versions. +- Consider implementing additional authentication mechanisms for the SMTP server if needed. + +## Logging and Monitoring + +The server logs information about received emails, webhook responses, and any errors that occur. The current logging setup includes: + +- Console output for immediate visibility +- Daily rotating log files for persistent storage +- JSON formatting of log entries for easy parsing +- Timestamp inclusion for each log entry + +Logging settings: + +- Log files are stored in the `logs/` directory +- Files are named `application-YYYY-MM-DD.log` +- Log files are rotated daily and compressed +- Maximum log file size is set to 20MB +- Log files are kept for 90 days + +I recommend: + +- Review log files regularly for errors or unusual patterns. +- Consider setting up log aggregation and analysis tools (e.g., ELK stack, Splunk). +- Implement alerts for critical errors or unusual activity patterns. +- Monitor system resources (CPU, memory, disk space) to ensure smooth operation. +- Set up uptime monitoring for the SMTP server and webhook endpoint. + +## System Requirements + +- Node.js v18 or later +- Sufficient disk space for temporary storage of attachments before S3 upload and 90 days of logging. +- Outbound internet access for S3 uploads and webhook calls +- Inbound access on the configured SMTP port. (Default: 25, or 587 if `SMTP_SECURE` is set to 'true') + +## Troubleshooting + +If you encounter issues: + +1. Check the server logs for any error messages. +2. Ensure all environment variables are correctly set. +3. Verify that your AWS credentials have the necessary permissions for S3 operations. +4. Check that the webhook endpoint is accessible and responding correctly. +5. For attachment issues, verify that the `MAX_FILE_SIZE` setting is appropriate for your use case. + +If problems persist, please open an issue on the GitHub repository with detailed information about the error and your setup. \ No newline at end of file diff --git a/server.js b/server.js index 4e588c2..ed9148a 100644 --- a/server.js +++ b/server.js @@ -3,21 +3,40 @@ const config = require('./config'); const { parseEmail } = require('./services/emailParser'); const { sendToWebhook } = require('./services/webhookService'); require('aws-sdk/lib/maintenance_mode_message').suppress = true; - -// Use a more robust queue implementation const Queue = require('better-queue'); +const winston = require('winston'); +require('winston-daily-rotate-file'); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.Console(), + new winston.transports.DailyRotateFile({ + filename: 'logs/application-%DATE%.log', + datePattern: 'YYYY-MM-DD', + zippedArchive: true, + maxSize: '20m', + maxFiles: '90d' + }) + ] +}); -// Create a queue for webhook sending const webhookQueue = new Queue(async function (parsed, cb) { try { await sendToWebhook(parsed); - console.log('Successfully sent to webhook'); + logger.info('Successfully sent to webhook'); cb(null); } catch (error) { - console.error('Webhook error:', error.message); + logger.error('Webhook error:', { message: error.message, stack: error.stack }); if (error.response) { - console.error('Response status:', error.response.status); - console.error('Response data:', error.response.data); + logger.error('Webhook response error:', { + status: error.response.status, + data: error.response.data + }); } cb(error); } @@ -28,16 +47,16 @@ const server = new SMTPServer({ parseEmail(stream) .then(parsed => { webhookQueue.push(parsed); - console.log('Email added to queue. Queue size:', webhookQueue.getStats().total); + logger.info('Email added to queue', { queueSize: webhookQueue.getStats().total }); callback(); }) .catch(error => { - console.error('Parsing error:', error); + logger.error('Parsing error:', { message: error.message, stack: error.stack }); callback(new Error('Failed to parse email')); }); }, onError(error) { - console.error('SMTP server error:', error); + logger.error('SMTP server error:', { message: error.message, stack: error.stack }); }, disabledCommands: ['AUTH'], secure: config.SMTP_SECURE @@ -45,27 +64,27 @@ const server = new SMTPServer({ server.listen(config.PORT, '0.0.0.0', err => { if (err) { - console.error('Failed to start SMTP server:', err); + logger.error('Failed to start SMTP server:', { message: err.message, stack: err.stack }); process.exit(1); } - console.log(`SMTP server listening on port ${config.PORT} on all interfaces`); + logger.info(`SMTP server listening on port ${config.PORT} on all interfaces`); }); function gracefulShutdown(reason) { - console.log(`Shutting down: ${reason}`); + logger.info(`Shutting down: ${reason}`); server.close(() => { - console.log('Server closed. Exiting process.'); + logger.info('Server closed. Exiting process.'); process.exit(0); }); } process.on('uncaughtException', (err) => { - console.error('Uncaught exception:', err); + logger.error('Uncaught exception:', { message: err.message, stack: err.stack }); gracefulShutdown('Uncaught exception'); }); process.on('unhandledRejection', (reason, promise) => { - console.error('Unhandled Rejection at:', promise, 'reason:', reason); + logger.error('Unhandled Rejection:', { reason: reason, promise: promise }); gracefulShutdown('Unhandled rejection'); }); From 9a71c58a90cd0a77572275576a87ab677f9d53c1 Mon Sep 17 00:00:00 2001 From: Martin <42486612+kriiv@users.noreply.github.com> Date: Sun, 22 Sep 2024 15:57:52 +1000 Subject: [PATCH 2/2] config validation & webhook retry --- server.js | 58 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/server.js b/server.js index ed9148a..f1555a0 100644 --- a/server.js +++ b/server.js @@ -25,21 +25,43 @@ const logger = winston.createLogger({ ] }); -const webhookQueue = new Queue(async function (parsed, cb) { - try { - await sendToWebhook(parsed); - logger.info('Successfully sent to webhook'); - cb(null); - } catch (error) { - logger.error('Webhook error:', { message: error.message, stack: error.stack }); - if (error.response) { - logger.error('Webhook response error:', { - status: error.response.status, - data: error.response.data - }); +function validateConfig() { + const requiredKeys = ['PORT', 'SMTP_SECURE', 'WEBHOOK_URL', 'WEBHOOK_CONCURRENCY']; + for (const key of requiredKeys) { + if (!(key in config)) { + throw new Error(`Missing required configuration: ${key}`); } - cb(error); } +} + +const webhookQueue = new Queue(async function (parsed, cb) { + const maxRetries = 3; + let retries = 0; + + const attemptWebhook = async () => { + try { + await sendToWebhook(parsed); + logger.info('Successfully sent to webhook'); + cb(null); + } catch (error) { + logger.error('Webhook error:', { message: error.message, stack: error.stack }); + if (error.response) { + logger.error('Webhook response error:', { + status: error.response.status, + data: error.response.data + }); + } + if (retries < maxRetries) { + retries++; + logger.info(`Retrying webhook (attempt ${retries}/${maxRetries})`); + setTimeout(attemptWebhook, 1000 * retries); + } else { + cb(error); + } + } + }; + + attemptWebhook(); }, { concurrent: config.WEBHOOK_CONCURRENCY || 5 }); const server = new SMTPServer({ @@ -94,4 +116,12 @@ process.on('SIGTERM', () => { process.on('SIGINT', () => { gracefulShutdown('SIGINT signal received'); -}); \ No newline at end of file +}); + +// Add configuration validation at startup +try { + validateConfig(); +} catch (error) { + logger.error('Configuration error:', { message: error.message }); + process.exit(1); +} \ No newline at end of file