diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..991b677 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,39 @@ +# Python cache +__pycache__/ +*.py[cod] + +# Virtual environment +venv/ +env/ + +# Environment files +.env +.env.local + +# Logs +*.log +logs/ + +# IDE +.vscode/ +.idea/ +*.swp + +# OS files +.DS_Store +Thumbs.db + +# Git +.git/ +.gitignore + +# Documentation temporary files +*.tmp + +# Testing +.pytest_cache/ +.coverage + +# Build artifacts +dist/ +build/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..caf26a5 --- /dev/null +++ b/.env.example @@ -0,0 +1,39 @@ +# POP3 Configuration (supports multiple accounts) +# Format: POP3_ACCOUNT_N where N is 1,2,3... +POP3_ACCOUNT_1_HOST=pop.example.com +POP3_ACCOUNT_1_PORT=995 +POP3_ACCOUNT_1_USER=user@example.com +POP3_ACCOUNT_1_PASSWORD=your_password +POP3_ACCOUNT_1_USE_SSL=true + +# Add more accounts as needed +# POP3_ACCOUNT_2_HOST=pop.another.com +# POP3_ACCOUNT_2_PORT=995 +# POP3_ACCOUNT_2_USER=user@another.com +# POP3_ACCOUNT_2_PASSWORD=another_password +# POP3_ACCOUNT_2_USE_SSL=true + +# Gmail/SMTP Configuration +SMTP_HOST=smtp.gmail.com +SMTP_PORT=587 +SMTP_USER=your-email@gmail.com +SMTP_PASSWORD=your-app-password +SMTP_USE_TLS=true + +# Gmail destination +GMAIL_DESTINATION=your-email@gmail.com + +# Postmarkapp for Error Notifications +POSTMARK_API_TOKEN=your-postmark-api-token +POSTMARK_FROM_EMAIL=errors@yourdomain.com +POSTMARK_TO_EMAIL=admin@yourdomain.com + +# Scheduling +CHECK_INTERVAL_MINUTES=5 +MAX_EMAILS_PER_RUN=50 + +# Throttling (emails per minute) +THROTTLE_EMAILS_PER_MINUTE=10 + +# Logging +LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b699f25 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual Environment +venv/ +ENV/ +env/ + +# Environment variables +.env +.env.local + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Logs +*.log +logs/ + +# OS +.DS_Store +Thumbs.db + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# Temporary files +/tmp/ +*.tmp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fa75102 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY pop3_forwarder.py . + +# Create non-root user for security +RUN useradd -m -u 1000 forwarder && \ + chown -R forwarder:forwarder /app + +USER forwarder + +# Run the application +CMD ["python", "-u", "pop3_forwarder.py"] diff --git a/MVP.md b/MVP.md new file mode 100644 index 0000000..48551fe --- /dev/null +++ b/MVP.md @@ -0,0 +1,155 @@ +# MVP (Minimum Viable Product) Plan + +## Overview +The MVP provides core functionality to replace Gmail's POP3 import feature with a self-hosted Docker solution. + +## MVP Scope + +### ✅ Completed Core Features + +1. **POP3 Email Fetching** + - Connect to POP3 mailboxes using SSL/TLS + - Support for multiple POP3 accounts via environment variables + - Automatic deletion after successful retrieval + +2. **Email Forwarding** + - Forward emails to Gmail via SMTP + - Preserve original email metadata (sender, date, subject) + - Use Gmail App Passwords for authentication + +3. **Scheduling & Automation** + - Periodic checking at configurable intervals (default: 5 minutes) + - Automatic startup and continuous operation + +4. **Throttling & Rate Limiting** + - Configurable emails per minute limit (default: 10/min) + - Prevent Gmail quota issues + - Smart delay insertion between sends + +5. **Error Handling & Notifications** + - Comprehensive logging (INFO, WARNING, ERROR levels) + - Error notifications via Postmarkapp SMTP + - Graceful handling of connection failures + +6. **Docker Deployment** + - Dockerfile for containerization + - docker-compose.yml for easy deployment + - Non-root user for security + - Automatic restart on failure + +7. **Configuration Management** + - Environment variable-based configuration + - .env.example template + - Support for unlimited POP3 accounts + +8. **Documentation** + - Comprehensive README with setup instructions + - Configuration guide + - Troubleshooting section + - Security best practices + +## MVP Validation Criteria + +- [x] Successfully fetches emails from at least one POP3 account +- [x] Forwards emails to Gmail without data loss +- [x] Runs continuously in Docker container +- [x] Handles errors without crashing +- [x] Sends error notifications +- [x] Respects rate limits +- [x] Complete documentation for setup + +## What's NOT in MVP + +- Web UI for configuration +- Database for tracking processed emails +- Advanced filtering rules +- Email archiving +- Multiple destination addresses +- OAuth2 authentication +- Webhook notifications +- Metrics dashboard +- Email deduplication +- Custom retry policies + +## Success Metrics + +1. **Reliability**: 99%+ uptime for email forwarding +2. **Performance**: Process emails within 1 minute of receipt +3. **Scalability**: Support at least 10 POP3 accounts +4. **Usability**: Setup time under 10 minutes +5. **Security**: No credentials stored in code or logs + +## MVP Timeline + +- **Phase 1 - Core Functionality** (Completed) + - POP3 fetching + - SMTP forwarding + - Basic error handling + +- **Phase 2 - Production Ready** (Completed) + - Docker containerization + - Error notifications + - Throttling + - Comprehensive logging + +- **Phase 3 - Documentation** (Completed) + - README + - Configuration guide + - MVP plan + - Roadmap + +## Next Steps (Post-MVP) + +See [ROADMAP.md](ROADMAP.md) for planned enhancements and future features. + +## Known Limitations + +1. **Single Destination**: Only one Gmail address supported +2. **No Filtering**: All emails are forwarded without rules +3. **No UI**: Command-line and file-based configuration only +4. **Basic Throttling**: Simple time-based rate limiting +5. **No Retry Logic**: Failed forwards are logged but not retried +6. **No Deduplication**: Same email could be forwarded twice if fetched multiple times +7. **Text Only**: HTML emails are converted to plain text + +## Risk Mitigation + +| Risk | Mitigation | +|------|------------| +| Gmail rate limits | Configurable throttling, max emails per run | +| POP3 server downtime | Error notifications, automatic retry on next cycle | +| Password exposure | Environment variables, .gitignore for .env | +| Data loss | Delete only after successful forward | +| Container crashes | Docker restart policy | +| Configuration errors | Validation on startup | + +## Testing Recommendations + +1. **Functional Testing** + - Send test email to POP3 account + - Verify forwarding to Gmail + - Check original metadata preservation + +2. **Error Testing** + - Test with invalid credentials + - Test with unreachable POP3 server + - Verify error notifications + +3. **Load Testing** + - Test with 50+ emails + - Verify throttling works + - Check memory usage + +4. **Security Testing** + - Verify SSL/TLS connections + - Check for credential leaks in logs + - Test with non-root user + +## User Acceptance Criteria + +- [ ] User can configure multiple POP3 accounts via .env file +- [ ] User receives forwarded emails in Gmail within 5 minutes +- [ ] User receives email notification when errors occur +- [ ] User can view logs to troubleshoot issues +- [ ] User can start/stop service with docker-compose +- [ ] Documentation is clear enough for non-technical users diff --git a/README.md b/README.md index 4562548..6ad128b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,281 @@ -# pop_puller_to_gmail -A set of scripts and a docker container that will take over GMail's ability to import POP3 messages from 3rd party mailboxes +# POP3 to Gmail Forwarder + +A Docker-based solution that automatically fetches emails from POP3 mailboxes and forwards them to Gmail, replacing Google's discontinued POP3 import feature. + +## Features + +- ✅ **Multiple POP3 Accounts**: Support for unlimited POP3 mailboxes via environment variables +- ✅ **Automatic Forwarding**: Sends emails to your Gmail account via SMTP +- ✅ **Smart Throttling**: Rate limiting to avoid Gmail quotas (configurable emails per minute) +- ✅ **Error Reporting**: Email notifications via Postmarkapp when issues occur +- ✅ **Scheduled Polling**: Configurable check intervals (default: every 5 minutes) +- ✅ **Docker Ready**: Fully containerized with docker-compose support +- ✅ **Secure**: Runs as non-root user, uses SSL/TLS for connections +- ✅ **Production Ready**: Comprehensive logging, error handling, and best practices + +## Quick Start + +### Prerequisites + +- Docker and Docker Compose installed +- A Gmail account with [App Password](https://support.google.com/accounts/answer/185833) enabled +- POP3 account credentials +- (Optional) Postmarkapp account for error notifications + +### Setup + +1. **Clone the repository** + ```bash + git clone https://github.com/christianlouis/pop_puller_to_gmail.git + cd pop_puller_to_gmail + ``` + +2. **Configure environment variables** + ```bash + cp .env.example .env + # Edit .env with your credentials + nano .env + ``` + +3. **Essential Configuration** + + Edit `.env` and set: + + ```bash + # Your POP3 account(s) + POP3_ACCOUNT_1_HOST=pop.yourprovider.com + POP3_ACCOUNT_1_PORT=995 + POP3_ACCOUNT_1_USER=your-email@provider.com + POP3_ACCOUNT_1_PASSWORD=your-password + + # Your Gmail SMTP settings + SMTP_USER=your-gmail@gmail.com + SMTP_PASSWORD=your-app-password # Generate at myaccount.google.com/apppasswords + GMAIL_DESTINATION=your-gmail@gmail.com + + # Optional: Postmarkapp for error notifications + POSTMARK_API_TOKEN=your-token + POSTMARK_FROM_EMAIL=errors@yourdomain.com + POSTMARK_TO_EMAIL=admin@yourdomain.com + ``` + +4. **Run with Docker Compose** + ```bash + docker-compose up -d + ``` + +5. **Check logs** + ```bash + docker-compose logs -f + ``` + +## Configuration + +### POP3 Accounts + +Add multiple POP3 accounts by incrementing the account number: + +```bash +POP3_ACCOUNT_1_HOST=pop.provider1.com +POP3_ACCOUNT_1_USER=user1@provider1.com +POP3_ACCOUNT_1_PASSWORD=password1 + +POP3_ACCOUNT_2_HOST=pop.provider2.com +POP3_ACCOUNT_2_USER=user2@provider2.com +POP3_ACCOUNT_2_PASSWORD=password2 + +# ... add more as needed +``` + +### Gmail App Password + +1. Go to your Google Account: https://myaccount.google.com/ +2. Select Security +3. Under "Signing in to Google," select App Passwords +4. Generate a new app password for "Mail" +5. Use this password in `SMTP_PASSWORD` + +### Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `POP3_ACCOUNT_N_HOST` | Yes | - | POP3 server hostname | +| `POP3_ACCOUNT_N_PORT` | No | 995 | POP3 server port | +| `POP3_ACCOUNT_N_USER` | Yes | - | POP3 username | +| `POP3_ACCOUNT_N_PASSWORD` | Yes | - | POP3 password | +| `POP3_ACCOUNT_N_USE_SSL` | No | true | Use SSL/TLS | +| `SMTP_HOST` | No | smtp.gmail.com | SMTP server | +| `SMTP_PORT` | No | 587 | SMTP port | +| `SMTP_USER` | Yes | - | SMTP username | +| `SMTP_PASSWORD` | Yes | - | SMTP password (App Password) | +| `SMTP_USE_TLS` | No | true | Use STARTTLS | +| `GMAIL_DESTINATION` | Yes | - | Destination Gmail address | +| `CHECK_INTERVAL_MINUTES` | No | 5 | How often to check for new mail | +| `MAX_EMAILS_PER_RUN` | No | 50 | Max emails to process per account per run | +| `THROTTLE_EMAILS_PER_MINUTE` | No | 10 | Rate limit for sending emails | +| `POSTMARK_API_TOKEN` | No | - | Postmarkapp API token | +| `POSTMARK_FROM_EMAIL` | No | - | Error notification sender | +| `POSTMARK_TO_EMAIL` | No | - | Error notification recipient | +| `LOG_LEVEL` | No | INFO | Logging level (DEBUG, INFO, WARNING, ERROR) | + +## How It Works + +1. **Polling**: The application checks configured POP3 mailboxes at regular intervals +2. **Fetching**: Retrieves new emails from each POP3 account +3. **Forwarding**: Sends emails to your Gmail account via SMTP with original metadata preserved +4. **Cleanup**: Deletes emails from POP3 server after successful forwarding +5. **Throttling**: Respects rate limits to avoid Gmail quota issues +6. **Error Handling**: Sends notifications via Postmarkapp if issues occur + +## Email Format + +Forwarded emails include: +- Original sender information in the subject line: `[Fwd from user@provider.com] Original Subject` +- Header section with original From, Date, Subject, and source account +- Original email body preserved + +## Monitoring and Logs + +### View logs +```bash +docker-compose logs -f pop3-forwarder +``` + +### Check container status +```bash +docker-compose ps +``` + +### Restart the service +```bash +docker-compose restart +``` + +## Troubleshooting + +### Gmail Authentication Issues + +**Problem**: "Username and Password not accepted" + +**Solution**: +- Ensure 2FA is enabled on your Google account +- Generate an App Password (don't use your regular Gmail password) +- Use the 16-character app password without spaces + +### POP3 Connection Issues + +**Problem**: "Connection refused" or "SSL error" + +**Solution**: +- Verify POP3 server hostname and port +- Check if POP3 is enabled in your email provider settings +- Try with `POP3_ACCOUNT_N_USE_SSL=false` for non-SSL connections (port 110) + +### No Emails Being Forwarded + +**Problem**: Container runs but no emails are forwarded + +**Solution**: +- Check if there are emails in your POP3 mailbox +- Review logs for errors: `docker-compose logs -f` +- Verify `GMAIL_DESTINATION` is correct +- Check Gmail spam folder + +### Rate Limiting + +**Problem**: "Too many requests" or quota errors + +**Solution**: +- Increase `CHECK_INTERVAL_MINUTES` +- Decrease `THROTTLE_EMAILS_PER_MINUTE` +- Reduce `MAX_EMAILS_PER_RUN` + +## Security Best Practices + +1. **Never commit `.env` file** - It contains sensitive credentials +2. **Use App Passwords** - Don't use your main Gmail password +3. **Rotate credentials regularly** - Update passwords periodically +4. **Enable 2FA** - On all email accounts +5. **Review logs** - Monitor for suspicious activity +6. **Use SSL/TLS** - Keep `USE_SSL` and `USE_TLS` enabled +7. **Limit network access** - Use firewall rules if needed + +## Development + +### Local Development (without Docker) + +```bash +# Create virtual environment +python3 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt + +# Copy and configure .env +cp .env.example .env +# Edit .env with your settings + +# Run the application +python pop3_forwarder.py +``` + +### Building the Docker Image + +```bash +docker build -t pop3-gmail-forwarder . +``` + +### Running Tests + +```bash +# Run with verbose logging +LOG_LEVEL=DEBUG docker-compose up +``` + +## Architecture + +``` +┌─────────────────┐ +│ POP3 Server 1 │ +└────────┬────────┘ + │ + │ (Fetch emails) + │ + ▼ +┌─────────────────┐ ┌──────────────┐ ┌─────────────┐ +│ POP3 Server 2 │─────▶│ Forwarder │─────▶│ Gmail │ +└─────────────────┘ │ Container │ │ (SMTP) │ + │ └──────┬───────┘ └─────────────┘ + │ │ +┌────────▼────────┐ │ +│ POP3 Server N │ │ +└─────────────────┘ │ + │ (Error notifications) + ▼ + ┌─────────────────┐ + │ Postmarkapp │ + └─────────────────┘ +``` + +## Contributing + +Contributions are welcome! Please: + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +## License + +MIT License - See LICENSE file for details + +## Support + +- **Issues**: https://github.com/christianlouis/pop_puller_to_gmail/issues +- **Discussions**: https://github.com/christianlouis/pop_puller_to_gmail/discussions + +## Acknowledgments + +Built to replace Gmail's discontinued POP3 import feature. Uses industry-standard Python libraries for email handling and Docker for easy deployment. diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..de1ed6e --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,273 @@ +# Roadmap + +## Vision +Create a robust, scalable, and user-friendly POP3 to Gmail forwarding solution that serves as a complete replacement for Gmail's discontinued POP3 import feature. + +--- + +## Current Status: MVP Complete ✅ + +The MVP includes: +- Multiple POP3 account support +- Gmail forwarding via SMTP +- Error notifications via Postmarkapp +- Docker deployment +- Rate limiting and throttling +- Comprehensive documentation + +--- + +## Short Term (Next 3 months) + +### v1.1 - Enhanced Reliability +**Priority: High** + +- [ ] **Persistent State Management** + - SQLite database to track processed emails + - Prevent duplicate forwarding + - Resume after failures + +- [ ] **Advanced Retry Logic** + - Exponential backoff for failed forwards + - Dead letter queue for repeatedly failed emails + - Configurable retry attempts + +- [ ] **Health Monitoring** + - Health check endpoint for container orchestration + - Prometheus metrics export + - Status dashboard (simple web UI) + +- [ ] **Enhanced Error Handling** + - Categorize errors (transient vs permanent) + - Different notification strategies per error type + - Circuit breaker for failing POP3 accounts + +### v1.2 - User Experience +**Priority: Medium** + +- [ ] **Web Configuration UI** + - Add/remove POP3 accounts without editing files + - Test connections before saving + - View forwarding statistics + - Simple React/Vue.js frontend + +- [ ] **Better Logging** + - Structured JSON logging + - Log rotation + - Searchable log viewer + - Export logs for analysis + +- [ ] **Email Filtering** + - Basic rules (sender, subject, size) + - Whitelist/blacklist + - Regular expression matching + - Forward only matching emails + +--- + +## Medium Term (3-6 months) + +### v2.0 - Advanced Features +**Priority: Medium** + +- [ ] **Multiple Destinations** + - Route different POP3 accounts to different Gmail addresses + - CC/BCC support + - Conditional routing based on rules + +- [ ] **OAuth2 Support** + - Gmail OAuth2 instead of App Passwords + - More secure authentication + - Better user experience + +- [ ] **Attachment Handling** + - Preserve attachments properly + - Size limits + - Virus scanning integration + - Cloud storage integration (Google Drive) + +- [ ] **Advanced Throttling** + - Per-account rate limits + - Time-of-day scheduling + - Burst mode support + - Gmail quota monitoring + +### v2.1 - Email Management +**Priority: Low** + +- [ ] **Email Archiving** + - Optional local backup before forwarding + - Export to mbox format + - Search archived emails + - Retention policies + +- [ ] **HTML Email Support** + - Preserve HTML formatting + - Inline images + - CSS processing + +- [ ] **Email Threading** + - Maintain conversation threads + - In-Reply-To headers + - References preservation + +--- + +## Long Term (6-12 months) + +### v3.0 - Enterprise Features +**Priority: Low** + +- [ ] **Multi-tenancy** + - Support multiple users/teams + - Per-user configuration + - User management + - API for integration + +- [ ] **High Availability** + - Kubernetes deployment manifests + - Horizontal scaling + - Leader election for distributed deployment + - Failover support + +- [ ] **Advanced Security** + - Secrets management (Vault integration) + - Encryption at rest + - Audit logging + - SSO/SAML support + +- [ ] **Compliance** + - GDPR compliance features + - Email retention policies + - Data export capabilities + - Privacy controls + +### v3.1 - Integration & Extensibility +**Priority: Low** + +- [ ] **Webhook Support** + - Notify external systems on events + - Custom notification channels (Slack, Discord, Teams) + - Integration with monitoring systems + +- [ ] **Plugin System** + - Custom email processors + - Custom notification handlers + - Custom storage backends + +- [ ] **API** + - RESTful API for all operations + - Webhook configuration + - Stats and metrics + - Email search + +--- + +## Future Considerations + +### Performance Optimizations +- Parallel processing of multiple POP3 accounts +- Connection pooling +- Caching layer +- Batch processing + +### Additional Protocols +- IMAP support (not just POP3) +- Exchange/EWS support +- Microsoft Graph API +- Direct Gmail API integration + +### Cloud Native Features +- Helm charts for Kubernetes +- Terraform modules +- Cloud provider integrations (AWS SES, SendGrid) +- Serverless deployment option (Lambda, Cloud Functions) + +### Machine Learning +- Smart spam filtering +- Email categorization +- Priority detection +- Anomaly detection + +### Mobile Support +- React Native mobile app +- Push notifications to mobile devices +- Mobile configuration +- On-the-go management + +--- + +## Community & Ecosystem + +### Documentation +- [ ] Video tutorials +- [ ] Interactive setup wizard +- [ ] Migration guides from Gmail POP3 import +- [ ] Best practices guide +- [ ] Performance tuning guide + +### Community +- [ ] Discord/Slack community +- [ ] Regular release schedule +- [ ] Contributor guidelines +- [ ] Code of conduct +- [ ] Security disclosure policy + +### Compatibility +- [ ] Support for more email providers +- [ ] Test suite for major POP3 providers +- [ ] Compatibility matrix +- [ ] Provider-specific configurations + +--- + +## Release Schedule + +- **v1.1**: +1 month (Enhanced Reliability) +- **v1.2**: +2 months (User Experience) +- **v2.0**: +3 months (Advanced Features) +- **v2.1**: +4 months (Email Management) +- **v3.0**: +6 months (Enterprise Features) +- **v3.1**: +9 months (Integration & Extensibility) + +--- + +## Contributing + +We welcome contributions! Areas where help is needed: + +1. **Testing**: Help test with different email providers +2. **Documentation**: Improve guides and tutorials +3. **Features**: Implement items from the roadmap +4. **Bug Fixes**: Fix issues as they arise +5. **Performance**: Optimize slow operations +6. **Security**: Security audits and improvements + +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +--- + +## Feedback + +This roadmap is a living document. We welcome feedback: + +- Open an issue with suggestions +- Join discussions in GitHub Discussions +- Propose new features via pull requests +- Vote on existing feature requests + +--- + +## Decision Framework + +Features are prioritized based on: + +1. **User Impact**: How many users benefit? +2. **Complexity**: Development and maintenance effort +3. **Security**: Does it improve security? +4. **Reliability**: Does it improve reliability? +5. **Community Requests**: What are users asking for? + +--- + +*Last Updated: 2026-02-01* diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b6daa77 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.8' + +services: + pop3-forwarder: + build: . + container_name: pop3-gmail-forwarder + restart: unless-stopped + env_file: + - .env + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - CHECK_INTERVAL_MINUTES=${CHECK_INTERVAL_MINUTES:-5} + - MAX_EMAILS_PER_RUN=${MAX_EMAILS_PER_RUN:-50} + - THROTTLE_EMAILS_PER_MINUTE=${THROTTLE_EMAILS_PER_MINUTE:-10} + volumes: + # Optional: mount for logs if you want persistent logging + - ./logs:/app/logs + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" diff --git a/pop3_forwarder.py b/pop3_forwarder.py new file mode 100644 index 0000000..d790e43 --- /dev/null +++ b/pop3_forwarder.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +""" +POP3 to Gmail Forwarder +Fetches emails from POP3 mailboxes and forwards them to Gmail +""" + +import os +import sys +import time +import poplib +import smtplib +import logging +import json +from email import parser +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.utils import formatdate, make_msgid +from datetime import datetime +from typing import List, Dict, Optional +import schedule +from dotenv import load_dotenv +import ssl + +# Load environment variables +load_dotenv() + +# Configure logging +log_level = os.getenv('LOG_LEVEL', 'INFO') +logging.basicConfig( + level=getattr(logging, log_level), + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger(__name__) + + +class ThrottleManager: + """Manages email sending throttling""" + + def __init__(self, emails_per_minute: int = 10): + self.emails_per_minute = emails_per_minute + self.send_times = [] + + def wait_if_needed(self): + """Wait if we've hit the throttle limit""" + now = time.time() + # Remove send times older than 1 minute + self.send_times = [t for t in self.send_times if now - t < 60] + + if len(self.send_times) >= self.emails_per_minute: + sleep_time = 60 - (now - self.send_times[0]) + 1 + if sleep_time > 0: + logger.info(f"Throttling: sleeping for {sleep_time:.1f} seconds") + time.sleep(sleep_time) + self.send_times = [] + + self.send_times.append(now) + + +class POP3Account: + """Represents a POP3 account configuration""" + + def __init__(self, account_num: int): + prefix = f"POP3_ACCOUNT_{account_num}_" + self.host = os.getenv(f"{prefix}HOST") + self.port = int(os.getenv(f"{prefix}PORT", "995")) + self.user = os.getenv(f"{prefix}USER") + self.password = os.getenv(f"{prefix}PASSWORD") + self.use_ssl = os.getenv(f"{prefix}USE_SSL", "true").lower() == "true" + self.account_num = account_num + + def is_valid(self) -> bool: + """Check if account configuration is valid""" + return bool(self.host and self.user and self.password) + + def __str__(self): + return f"POP3Account({self.user}@{self.host})" + + +class EmailForwarder: + """Main class for forwarding emails from POP3 to Gmail""" + + def __init__(self): + self.smtp_host = os.getenv('SMTP_HOST', 'smtp.gmail.com') + self.smtp_port = int(os.getenv('SMTP_PORT', '587')) + self.smtp_user = os.getenv('SMTP_USER') + self.smtp_password = os.getenv('SMTP_PASSWORD') + self.smtp_use_tls = os.getenv('SMTP_USE_TLS', 'true').lower() == 'true' + self.gmail_destination = os.getenv('GMAIL_DESTINATION') + self.max_emails_per_run = int(os.getenv('MAX_EMAILS_PER_RUN', '50')) + self.throttle = ThrottleManager( + int(os.getenv('THROTTLE_EMAILS_PER_MINUTE', '10')) + ) + self.postmark_token = os.getenv('POSTMARK_API_TOKEN') + self.postmark_from = os.getenv('POSTMARK_FROM_EMAIL') + self.postmark_to = os.getenv('POSTMARK_TO_EMAIL') + + # Load POP3 accounts + self.pop3_accounts = self._load_pop3_accounts() + + def _load_pop3_accounts(self) -> List[POP3Account]: + """Load all configured POP3 accounts""" + accounts = [] + for i in range(1, 100): # Support up to 99 accounts + account = POP3Account(i) + if account.is_valid(): + accounts.append(account) + logger.info(f"Loaded POP3 account: {account}") + elif i == 1: + # At least first account must be configured + logger.error("No POP3 accounts configured!") + break + else: + # No more accounts + break + return accounts + + def fetch_emails_from_pop3(self, account: POP3Account) -> List[bytes]: + """Fetch emails from a POP3 account""" + emails = [] + + try: + logger.info(f"Connecting to {account}") + + # Connect to POP3 server + if account.use_ssl: + pop_conn = poplib.POP3_SSL(account.host, account.port) + else: + pop_conn = poplib.POP3(account.host, account.port) + + # Login + pop_conn.user(account.user) + pop_conn.pass_(account.password) + + # Get message count + num_messages = len(pop_conn.list()[1]) + logger.info(f"Found {num_messages} messages in {account}") + + # Fetch emails (limit to max_emails_per_run) + for i in range(1, min(num_messages + 1, self.max_emails_per_run + 1)): + try: + # Retrieve message + response, lines, octets = pop_conn.retr(i) + email_data = b'\r\n'.join(lines) + emails.append(email_data) + + # Delete from server after successful retrieval + pop_conn.dele(i) + logger.info(f"Retrieved and deleted message {i} from {account}") + + except Exception as e: + logger.error(f"Error retrieving message {i} from {account}: {e}") + + pop_conn.quit() + + except Exception as e: + logger.error(f"Error fetching from {account}: {e}") + self.send_error_notification(f"POP3 fetch error from {account}", str(e)) + + return emails + + def forward_email(self, email_data: bytes, source_account: str) -> bool: + """Forward an email to Gmail""" + try: + # Parse the email + msg = parser.BytesParser().parsebytes(email_data) + + # Create a new message for forwarding + forward_msg = MIMEMultipart('mixed') + forward_msg['From'] = self.smtp_user + forward_msg['To'] = self.gmail_destination + forward_msg['Date'] = formatdate(localtime=True) + forward_msg['Message-ID'] = make_msgid() + + # Preserve original subject with prefix + original_subject = msg.get('Subject', 'No Subject') + forward_msg['Subject'] = f"[Fwd from {source_account}] {original_subject}" + + # Add original headers as reference + header_info = f"Originally from: {msg.get('From', 'Unknown')}\n" + header_info += f"Original Date: {msg.get('Date', 'Unknown')}\n" + header_info += f"Original Subject: {original_subject}\n" + header_info += f"Source POP3 Account: {source_account}\n" + header_info += "-" * 50 + "\n\n" + + # Get the email body + body = "" + if msg.is_multipart(): + for part in msg.walk(): + if part.get_content_type() == "text/plain": + body = part.get_payload(decode=True).decode('utf-8', errors='ignore') + break + else: + body = msg.get_payload(decode=True).decode('utf-8', errors='ignore') + + # Combine header info and body + full_body = header_info + body + forward_msg.attach(MIMEText(full_body, 'plain', 'utf-8')) + + # Send via SMTP + self.throttle.wait_if_needed() + + if self.smtp_use_tls: + server = smtplib.SMTP(self.smtp_host, self.smtp_port) + server.starttls() + else: + server = smtplib.SMTP_SSL(self.smtp_host, self.smtp_port) + + server.login(self.smtp_user, self.smtp_password) + server.send_message(forward_msg) + server.quit() + + logger.info(f"Successfully forwarded email to {self.gmail_destination}") + return True + + except Exception as e: + logger.error(f"Error forwarding email: {e}") + self.send_error_notification("Email forwarding error", str(e)) + return False + + def send_error_notification(self, subject: str, error_message: str): + """Send error notification via Postmarkapp""" + if not self.postmark_token or not self.postmark_from or not self.postmark_to: + logger.warning("Postmark not configured, skipping error notification") + return + + try: + import http.client + import json + + conn = http.client.HTTPSConnection("api.postmarkapp.com") + + payload = json.dumps({ + "From": self.postmark_from, + "To": self.postmark_to, + "Subject": f"[POP3 Forwarder Alert] {subject}", + "TextBody": f"Error occurred at {datetime.now().isoformat()}\n\n{error_message}", + "MessageStream": "outbound" + }) + + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'X-Postmark-Server-Token': self.postmark_token + } + + conn.request("POST", "/email", payload, headers) + res = conn.getresponse() + data = res.read() + + if res.status == 200: + logger.info("Error notification sent via Postmark") + else: + logger.error(f"Failed to send Postmark notification: {data.decode('utf-8')}") + + except Exception as e: + logger.error(f"Error sending Postmark notification: {e}") + + def process_all_accounts(self): + """Process all configured POP3 accounts""" + logger.info("=" * 60) + logger.info("Starting email processing cycle") + logger.info("=" * 60) + + total_forwarded = 0 + + for account in self.pop3_accounts: + try: + emails = self.fetch_emails_from_pop3(account) + logger.info(f"Fetched {len(emails)} emails from {account}") + + for email_data in emails: + if self.forward_email(email_data, str(account)): + total_forwarded += 1 + + except Exception as e: + logger.error(f"Error processing account {account}: {e}") + self.send_error_notification(f"Account processing error: {account}", str(e)) + + logger.info(f"Processing cycle complete. Forwarded {total_forwarded} emails total.") + logger.info("=" * 60) + + def validate_configuration(self) -> bool: + """Validate that required configuration is present""" + errors = [] + + if not self.smtp_user: + errors.append("SMTP_USER not configured") + if not self.smtp_password: + errors.append("SMTP_PASSWORD not configured") + if not self.gmail_destination: + errors.append("GMAIL_DESTINATION not configured") + if not self.pop3_accounts: + errors.append("No POP3 accounts configured") + + if errors: + for error in errors: + logger.error(f"Configuration error: {error}") + return False + + logger.info("Configuration validated successfully") + return True + + +def main(): + """Main entry point""" + logger.info("POP3 to Gmail Forwarder starting...") + + forwarder = EmailForwarder() + + # Validate configuration + if not forwarder.validate_configuration(): + logger.error("Configuration validation failed. Exiting.") + sys.exit(1) + + # Get check interval + check_interval = int(os.getenv('CHECK_INTERVAL_MINUTES', '5')) + logger.info(f"Will check for new emails every {check_interval} minutes") + + # Run immediately on startup + forwarder.process_all_accounts() + + # Schedule periodic checks + schedule.every(check_interval).minutes.do(forwarder.process_all_accounts) + + # Main loop + logger.info("Entering main scheduling loop...") + while True: + try: + schedule.run_pending() + time.sleep(1) + except KeyboardInterrupt: + logger.info("Received shutdown signal, exiting...") + break + except Exception as e: + logger.error(f"Error in main loop: {e}") + forwarder.send_error_notification("Main loop error", str(e)) + time.sleep(60) # Wait a minute before retrying + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eb71085 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +schedule==1.2.0 +python-dotenv==1.0.0