feat: show mail import history
This commit is contained in:
@@ -73,8 +73,10 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
|
|||||||
# See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
|
# See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
|
||||||
csp_directives = [
|
csp_directives = [
|
||||||
"default-src 'self'",
|
"default-src 'self'",
|
||||||
# TODO: Remove 'unsafe-inline' - requires moving inline scripts to external files # pylint: disable=fixme
|
# TODO: Remove 'unsafe-inline' and 'unsafe-eval' - requires moving inline
|
||||||
"script-src 'self' 'unsafe-inline'"
|
# scripts to external files and replacing the standard Alpine CDN build
|
||||||
|
# with the CSP-compatible build. # pylint: disable=fixme
|
||||||
|
"script-src 'self' 'unsafe-inline' 'unsafe-eval'"
|
||||||
" https://cdn.tailwindcss.com https://cdn.jsdelivr.net",
|
" https://cdn.tailwindcss.com https://cdn.jsdelivr.net",
|
||||||
# TODO: Remove 'unsafe-inline' - requires moving inline styles to CSS or using nonces # pylint: disable=fixme
|
# TODO: Remove 'unsafe-inline' - requires moving inline styles to CSS or using nonces # pylint: disable=fixme
|
||||||
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com"
|
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com"
|
||||||
|
|||||||
@@ -105,6 +105,14 @@
|
|||||||
<polyline points="22 4 12 14.01 9 11.01"></polyline>
|
<polyline points="22 4 12 14.01 9 11.01"></polyline>
|
||||||
</svg>
|
</svg>
|
||||||
</button>
|
</button>
|
||||||
|
<button class="btn btn-ghost btn-xs" x-on:click="loadImportHistory(source)" title="Import history">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24"
|
||||||
|
fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
|
||||||
|
stroke-linejoin="round">
|
||||||
|
<circle cx="12" cy="12" r="10"></circle>
|
||||||
|
<polyline points="12 6 12 12 16 14"></polyline>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
<button class="btn btn-ghost btn-xs text-error" x-on:click="confirmDelete(source)" title="Delete">
|
<button class="btn btn-ghost btn-xs text-error" x-on:click="confirmDelete(source)" title="Delete">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24"
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24"
|
||||||
fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
|
fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
|
||||||
@@ -127,6 +135,79 @@
|
|||||||
{% endcall %}
|
{% endcall %}
|
||||||
{% endcall %}
|
{% endcall %}
|
||||||
|
|
||||||
|
<!-- Import history -->
|
||||||
|
<template x-if="historySource">
|
||||||
|
{% call card() %}
|
||||||
|
{% call card_header() %}
|
||||||
|
<div class="flex items-start justify-between gap-4">
|
||||||
|
<div>
|
||||||
|
{% call card_title() %}Import History{% endcall %}
|
||||||
|
{% call card_description() %}
|
||||||
|
<span x-text="historySource ? historySource.name : ''"></span>
|
||||||
|
{% endcall %}
|
||||||
|
</div>
|
||||||
|
<button class="btn btn-ghost btn-sm" x-on:click="closeHistory()">Close</button>
|
||||||
|
</div>
|
||||||
|
{% endcall %}
|
||||||
|
{% call card_content() %}
|
||||||
|
<template x-if="historyLoading">
|
||||||
|
<p class="text-muted-foreground text-sm py-4 text-center">Loading import history…</p>
|
||||||
|
</template>
|
||||||
|
<template x-if="!historyLoading && importHistory.length === 0">
|
||||||
|
<p class="text-muted-foreground text-sm py-4 text-center">No import attempts recorded yet.</p>
|
||||||
|
</template>
|
||||||
|
<template x-if="!historyLoading && importHistory.length > 0">
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
<table class="table w-full">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Started</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Trigger</th>
|
||||||
|
<th>Emails</th>
|
||||||
|
<th>Reports</th>
|
||||||
|
<th>Duplicates</th>
|
||||||
|
<th>New Domains</th>
|
||||||
|
<th>Errors</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<template x-for="entry in importHistory" :key="entry.id">
|
||||||
|
<tr>
|
||||||
|
<td x-text="formatDate(entry.started_at)"></td>
|
||||||
|
<td>
|
||||||
|
<span class="badge" :class="statusBadgeClass(entry.status)" x-text="entry.status"></span>
|
||||||
|
</td>
|
||||||
|
<td x-text="entry.trigger"></td>
|
||||||
|
<td x-text="entry.processed"></td>
|
||||||
|
<td x-text="entry.reports_found"></td>
|
||||||
|
<td x-text="entry.duplicate_reports"></td>
|
||||||
|
<td x-text="formatList(entry.new_domains)"></td>
|
||||||
|
<td>
|
||||||
|
<template x-if="entry.errors && entry.errors.length">
|
||||||
|
<details class="max-w-xs">
|
||||||
|
<summary class="cursor-pointer" x-text="`${entry.error_count} error${entry.error_count === 1 ? '' : 's'}`"></summary>
|
||||||
|
<ul class="list-disc pl-4 mt-2 text-xs space-y-1">
|
||||||
|
<template x-for="error in entry.errors" :key="error">
|
||||||
|
<li x-text="error"></li>
|
||||||
|
</template>
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
</template>
|
||||||
|
<template x-if="!entry.errors || entry.errors.length === 0">
|
||||||
|
<span class="text-muted-foreground">None</span>
|
||||||
|
</template>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</template>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
{% endcall %}
|
||||||
|
{% endcall %}
|
||||||
|
</template>
|
||||||
|
|
||||||
<!-- Add / Edit modal -->
|
<!-- Add / Edit modal -->
|
||||||
<div x-show="showForm" x-cloak
|
<div x-show="showForm" x-cloak
|
||||||
class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4"
|
class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4"
|
||||||
@@ -390,6 +471,9 @@ function mailSourcesApp() {
|
|||||||
isTesting: false,
|
isTesting: false,
|
||||||
isSaving: false,
|
isSaving: false,
|
||||||
testing: {},
|
testing: {},
|
||||||
|
historySource: null,
|
||||||
|
importHistory: [],
|
||||||
|
historyLoading: false,
|
||||||
feedback: { message: '', type: '' },
|
feedback: { message: '', type: '' },
|
||||||
testResult: { message: '', success: false },
|
testResult: { message: '', success: false },
|
||||||
gmailConnected: false,
|
gmailConnected: false,
|
||||||
@@ -425,6 +509,46 @@ function mailSourcesApp() {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
async loadImportHistory(source) {
|
||||||
|
this.historySource = source;
|
||||||
|
this.importHistory = [];
|
||||||
|
this.historyLoading = true;
|
||||||
|
this.feedback = { message: '', type: '' };
|
||||||
|
try {
|
||||||
|
const resp = await fetch(`/api/v1/mail-sources/${source.id}/imports?limit=20`);
|
||||||
|
if (!resp.ok) {
|
||||||
|
const err = await resp.json();
|
||||||
|
throw new Error(err.detail || 'Failed to load import history');
|
||||||
|
}
|
||||||
|
this.importHistory = await resp.json();
|
||||||
|
} catch (e) {
|
||||||
|
this.feedback = { message: `Error: ${e.message}`, type: 'error' };
|
||||||
|
} finally {
|
||||||
|
this.historyLoading = false;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
closeHistory() {
|
||||||
|
this.historySource = null;
|
||||||
|
this.importHistory = [];
|
||||||
|
this.historyLoading = false;
|
||||||
|
},
|
||||||
|
|
||||||
|
formatDate(value) {
|
||||||
|
return value ? new Date(value).toLocaleString() : '—';
|
||||||
|
},
|
||||||
|
|
||||||
|
formatList(value) {
|
||||||
|
return value && value.length ? value.join(', ') : '—';
|
||||||
|
},
|
||||||
|
|
||||||
|
statusBadgeClass(status) {
|
||||||
|
if (status === 'success') return 'badge-success';
|
||||||
|
if (status === 'warning') return 'badge-warning';
|
||||||
|
if (status === 'failed') return 'badge-error';
|
||||||
|
return 'badge-outline';
|
||||||
|
},
|
||||||
|
|
||||||
openAddForm() {
|
openAddForm() {
|
||||||
this.editingId = null;
|
this.editingId = null;
|
||||||
this.gmailConnected = false;
|
this.gmailConnected = false;
|
||||||
|
|||||||
@@ -123,6 +123,18 @@ class TestFileUploadSecurity:
|
|||||||
DMARCParser.parse_file(large_content, "test.xml")
|
DMARCParser.parse_file(large_content, "test.xml")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSecurityHeaders:
|
||||||
|
"""Test security headers that affect the browser UI."""
|
||||||
|
|
||||||
|
def test_csp_allows_current_alpine_runtime(self, client: TestClient):
|
||||||
|
"""The current Alpine CDN build needs eval permission to render UI pages."""
|
||||||
|
response = client.get("/mail-sources")
|
||||||
|
csp = response.headers["Content-Security-Policy"]
|
||||||
|
|
||||||
|
assert "'unsafe-eval'" in csp
|
||||||
|
assert "https://cdn.jsdelivr.net" in csp
|
||||||
|
|
||||||
|
|
||||||
class TestXMLParsingSecurity:
|
class TestXMLParsingSecurity:
|
||||||
"""Test XML parsing security (defusedxml, XXE protection)."""
|
"""Test XML parsing security (defusedxml, XXE protection)."""
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ Recently improved:
|
|||||||
- Mail source imports now create sanitized import-history records for manual and scheduled polls.
|
- Mail source imports now create sanitized import-history records for manual and scheduled polls.
|
||||||
- Parsed upload, Gmail, and IMAP reports are now persisted to `dmarc_reports` and `report_records`.
|
- Parsed upload, Gmail, and IMAP reports are now persisted to `dmarc_reports` and `report_records`.
|
||||||
- Report/domain API reads can hydrate the dashboard projection from persisted data after restart.
|
- Report/domain API reads can hydrate the dashboard projection from persisted data after restart.
|
||||||
|
- Mail source import history is visible in the Mail Sources UI.
|
||||||
|
- The current Alpine-based UI is allowed by CSP and renders dynamic tables in real browsers.
|
||||||
|
|
||||||
Implementation note:
|
Implementation note:
|
||||||
- The legacy `ReportStore` remains as a projection layer for existing report/dashboard code, but durable report data now lives in the database.
|
- The legacy `ReportStore` remains as a projection layer for existing report/dashboard code, but durable report data now lives in the database.
|
||||||
@@ -36,7 +38,6 @@ Objective: make mailbox imports auditable and make report totals trustworthy.
|
|||||||
|
|
||||||
Priority tasks:
|
Priority tasks:
|
||||||
- Expand import attempts with message ID, source, attachment filename, outcome, and sanitized error details.
|
- Expand import attempts with message ID, source, attachment filename, outcome, and sanitized error details.
|
||||||
- Show import history on mail source detail pages.
|
|
||||||
- Report duplicate skips separately from parse failures.
|
- Report duplicate skips separately from parse failures.
|
||||||
- Add backfill controls for Gmail and IMAP sources.
|
- Add backfill controls for Gmail and IMAP sources.
|
||||||
- Improve source rollups so a source IP tracks pass/fail counts over time.
|
- Improve source rollups so a source IP tracks pass/fail counts over time.
|
||||||
|
|||||||
+2
-1
@@ -61,10 +61,11 @@ Recently delivered:
|
|||||||
- Tests now cover a real Google-style ZIP attachment path rather than only mocked parser behavior.
|
- Tests now cover a real Google-style ZIP attachment path rather than only mocked parser behavior.
|
||||||
- Mail source imports now persist sanitized import-history records for manual and scheduled polls.
|
- Mail source imports now persist sanitized import-history records for manual and scheduled polls.
|
||||||
- Uploaded, Gmail-imported, and IMAP-imported reports are now persisted to report/record tables and can be reloaded into report/domain views.
|
- Uploaded, Gmail-imported, and IMAP-imported reports are now persisted to report/record tables and can be reloaded into report/domain views.
|
||||||
|
- Mail source import history is now visible from the Mail Sources UI.
|
||||||
|
- The current Alpine-based UI can run under the configured CSP, so dynamic tables render in real browsers.
|
||||||
|
|
||||||
Next tasks:
|
Next tasks:
|
||||||
- Add per-import result details: skipped duplicates, parse failures, unsupported attachments, and imported report IDs.
|
- Add per-import result details: skipped duplicates, parse failures, unsupported attachments, and imported report IDs.
|
||||||
- Add a UI import history view for each mail source.
|
|
||||||
- Add mailbox search controls for date range/backfill without requiring code changes.
|
- Add mailbox search controls for date range/backfill without requiring code changes.
|
||||||
- Improve source aggregation so each sender IP keeps pass/fail totals instead of only the latest result.
|
- Improve source aggregation so each sender IP keeps pass/fail totals instead of only the latest result.
|
||||||
|
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA
|
|||||||
- [x] Skip duplicate domain/report IDs during IMAP imports
|
- [x] Skip duplicate domain/report IDs during IMAP imports
|
||||||
- [x] Persist sanitized import errors for API/UI review
|
- [x] Persist sanitized import errors for API/UI review
|
||||||
- [x] Count duplicate skips separately from parse failures
|
- [x] Count duplicate skips separately from parse failures
|
||||||
|
- [x] Show recent import history in the Mail Sources UI
|
||||||
- [ ] Add retry/backfill controls per mail source
|
- [ ] Add retry/backfill controls per mail source
|
||||||
|
|
||||||
## Milestone 3: Database Integration
|
## Milestone 3: Database Integration
|
||||||
|
|||||||
Reference in New Issue
Block a user