Merge pull request #1 from sendbetter/dev

added logging, config validation & updated readme

package.json

@@ -10,6 +10,8 @@
         "better-queue": "^3.8.12",
         "dotenv": "^16.4.5",
         "mailparser": "^3.7.1",
-        "smtp-server": "^3.13.5"
+        "smtp-server": "^3.13.5",
+        "winston": "^3.x.x",
+        "winston-daily-rotate-file": "^4.x.x"
     }
 }

readme.md

@@ -72,11 +72,11 @@ Using inbound parse for something interesting? Please let me know, I'd love to h
 ## Todo
 
 - Rate limiting
-- Log Storage
+- ~~Log Storage~~ (completed)
 
 ## Contributing
 
-Contributions are welcome! Please feel free to submit a Pull Request.
+Contributions are welcome! Please feel free to submit a Pull Request or get in touch.  
 
 ## License
 
@@ -85,3 +85,56 @@ This project is licensed under the MIT License. This means you are free to use,
 ## Disclaimer
 
 Please ensure you have the necessary permissions and security measures in place when deploying an SMTP server. Depending on your firewall configuration, you may be exposing this service to the internet.
+
+## Security Considerations
+
+When deploying this SMTP server, please keep the following security considerations in mind:
+
+- Ensure that your server is properly secured and that only authorized IPs can access the SMTP port.
+- Use strong, unique passwords for your AWS credentials and keep them secure.
+- Regularly update the Node.js runtime and all dependencies to their latest versions.
+- Consider implementing additional authentication mechanisms for the SMTP server if needed.
+
+## Logging and Monitoring
+
+The server logs information about received emails, webhook responses, and any errors that occur. The current logging setup includes:
+
+- Console output for immediate visibility
+- Daily rotating log files for persistent storage
+- JSON formatting of log entries for easy parsing
+- Timestamp inclusion for each log entry
+ 
+Logging settings:
+
+- Log files are stored in the `logs/` directory
+- Files are named `application-YYYY-MM-DD.log`
+- Log files are rotated daily and compressed
+- Maximum log file size is set to 20MB
+- Log files are kept for 90 days
+
+I recommend:   
+
+- Review log files regularly for errors or unusual patterns.
+- Consider setting up log aggregation and analysis tools (e.g., ELK stack, Splunk).
+- Implement alerts for critical errors or unusual activity patterns.
+- Monitor system resources (CPU, memory, disk space) to ensure smooth operation.
+- Set up uptime monitoring for the SMTP server and webhook endpoint.
+
+## System Requirements
+
+- Node.js v18 or later
+- Sufficient disk space for temporary storage of attachments before S3 upload and 90 days of logging.
+- Outbound internet access for S3 uploads and webhook calls
+- Inbound access on the configured SMTP port. (Default: 25, or 587 if `SMTP_SECURE` is set to 'true')
+
+## Troubleshooting
+
+If you encounter issues:
+
+1. Check the server logs for any error messages.
+2. Ensure all environment variables are correctly set.
+3. Verify that your AWS credentials have the necessary permissions for S3 operations.
+4. Check that the webhook endpoint is accessible and responding correctly.
+5. For attachment issues, verify that the `MAX_FILE_SIZE` setting is appropriate for your use case.
+
+If problems persist, please open an issue on the GitHub repository with detailed information about the error and your setup.

server.js

@@ -3,24 +3,65 @@ const config = require('./config');
 const { parseEmail } = require('./services/emailParser');
 const { sendToWebhook } = require('./services/webhookService');
 require('aws-sdk/lib/maintenance_mode_message').suppress = true;
-
-// Use a more robust queue implementation
 const Queue = require('better-queue');
+const winston = require('winston');
+require('winston-daily-rotate-file');
+
+const logger = winston.createLogger({
+  level: 'info',
+  format: winston.format.combine(
+    winston.format.timestamp(),
+    winston.format.json()
+  ),
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.DailyRotateFile({
+      filename: 'logs/application-%DATE%.log',
+      datePattern: 'YYYY-MM-DD',
+      zippedArchive: true,
+      maxSize: '20m',
+      maxFiles: '90d'
+    })
+  ]
+});
+
+function validateConfig() {
+  const requiredKeys = ['PORT', 'SMTP_SECURE', 'WEBHOOK_URL', 'WEBHOOK_CONCURRENCY'];
+  for (const key of requiredKeys) {
+    if (!(key in config)) {
+      throw new Error(`Missing required configuration: ${key}`);
+    }
+  }
+}
 
-// Create a queue for webhook sending
 const webhookQueue = new Queue(async function (parsed, cb) {
+  const maxRetries = 3;
+  let retries = 0;
+
+  const attemptWebhook = async () => {
     try {
       await sendToWebhook(parsed);
-    console.log('Successfully sent to webhook');
+      logger.info('Successfully sent to webhook');
       cb(null);
     } catch (error) {
-    console.error('Webhook error:', error.message);
+      logger.error('Webhook error:', { message: error.message, stack: error.stack });
       if (error.response) {
-      console.error('Response status:', error.response.status);
+        logger.error('Webhook response error:', { 
-      console.error('Response data:', error.response.data);
+          status: error.response.status, 
+          data: error.response.data 
+        });
       }
+      if (retries < maxRetries) {
+        retries++;
+        logger.info(`Retrying webhook (attempt ${retries}/${maxRetries})`);
+        setTimeout(attemptWebhook, 1000 * retries);
+      } else {
         cb(error);
       }
+    }
+  };
+
+  attemptWebhook();
 }, { concurrent: config.WEBHOOK_CONCURRENCY || 5 });
 
 const server = new SMTPServer({
@@ -28,16 +69,16 @@ const server = new SMTPServer({
     parseEmail(stream)
       .then(parsed => {
         webhookQueue.push(parsed);
-        console.log('Email added to queue. Queue size:', webhookQueue.getStats().total);
+        logger.info('Email added to queue', { queueSize: webhookQueue.getStats().total });
         callback();
       })
       .catch(error => {
-        console.error('Parsing error:', error);
+        logger.error('Parsing error:', { message: error.message, stack: error.stack });
         callback(new Error('Failed to parse email'));
       });
   },
   onError(error) {
-    console.error('SMTP server error:', error);
+    logger.error('SMTP server error:', { message: error.message, stack: error.stack });
   },
   disabledCommands: ['AUTH'],
   secure: config.SMTP_SECURE
@@ -45,27 +86,27 @@ const server = new SMTPServer({
 
 server.listen(config.PORT, '0.0.0.0', err => {
   if (err) {
-    console.error('Failed to start SMTP server:', err);
+    logger.error('Failed to start SMTP server:', { message: err.message, stack: err.stack });
     process.exit(1);
   }
-  console.log(`SMTP server listening on port ${config.PORT} on all interfaces`);
+  logger.info(`SMTP server listening on port ${config.PORT} on all interfaces`);
 });
 
 function gracefulShutdown(reason) {
-  console.log(`Shutting down: ${reason}`);
+  logger.info(`Shutting down: ${reason}`);
   server.close(() => {
-    console.log('Server closed. Exiting process.');
+    logger.info('Server closed. Exiting process.');
     process.exit(0);
   });
 }
 
 process.on('uncaughtException', (err) => {
-  console.error('Uncaught exception:', err);
+  logger.error('Uncaught exception:', { message: err.message, stack: err.stack });
   gracefulShutdown('Uncaught exception');
 });
 
 process.on('unhandledRejection', (reason, promise) => {
-  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  logger.error('Unhandled Rejection:', { reason: reason, promise: promise });
   gracefulShutdown('Unhandled rejection');
 });
 
@@ -76,3 +117,11 @@ process.on('SIGTERM', () => {
 process.on('SIGINT', () => {
   gracefulShutdown('SIGINT signal received');
 });
+
+// Add configuration validation at startup
+try {
+  validateConfig();
+} catch (error) {
+  logger.error('Configuration error:', { message: error.message });
+  process.exit(1);
+}