From 144409502bf48c738d23b5501cfd65983723e78d Mon Sep 17 00:00:00 2001 From: alma Date: Thu, 1 Jan 2026 19:05:25 +0100 Subject: [PATCH] improve courrier --- COURRIER_USER_MANAGEMENT.md | 304 ++++++++++++++++++++++++++++++ app/api/courrier/account/route.ts | 68 ++++++- app/api/courrier/session/route.ts | 66 ++++++- 3 files changed, 430 insertions(+), 8 deletions(-) create mode 100644 COURRIER_USER_MANAGEMENT.md diff --git a/COURRIER_USER_MANAGEMENT.md b/COURRIER_USER_MANAGEMENT.md new file mode 100644 index 00000000..5ee71813 --- /dev/null +++ b/COURRIER_USER_MANAGEMENT.md @@ -0,0 +1,304 @@ +# Courrier User Management with Prisma + +## Overview + +**Important**: Courrier (the email system) does **NOT** create User records in Prisma. It only manages email account credentials (`MailCredentials`) for users that already exist in the database. + +## User Creation Flow + +### 1. User Creation in Keycloak (Primary Source) + +Users are created in **Keycloak** first, which is the primary authentication system: + +**Location**: `app/api/users/route.ts` (POST method) + +**Process**: +1. User is created in Keycloak via Admin API +2. Roles are assigned to the user +3. User may be created in external systems: + - **Leantime** (project management tool) + - **Dolibarr** (if user has "Mediation" or "Expression" roles) + +**Key Code**: +```typescript +// Create user in Keycloak +const createResponse = await fetch( + `${process.env.KEYCLOAK_BASE_URL}/admin/realms/${process.env.KEYCLOAK_REALM}/users`, + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + username: data.username, + enabled: true, + emailVerified: true, + firstName: data.firstName, + lastName: data.lastName, + email: data.email, + credentials: [{ type: "password", value: data.password, temporary: false }], + }), + } +); +``` + +### 2. User Sync to Prisma Database + +After creation in Keycloak, users need to be synced to the Prisma database. This happens via: + +**Option A: Manual Sync Script** +- `scripts/sync-users.ts` or `scripts/sync-users.js` +- Fetches users from Keycloak API +- Creates/updates User records in Prisma + +**Option B: API Endpoint** +- `app/api/sync-users/route.ts` (GET method) +- Can be called to sync users programmatically + +**Prisma User Creation**: +```typescript +await prisma.user.create({ + data: { + id: user.id, // Use the Keycloak ID as primary ID + email: user.email, + password: tempPassword, // Temporary password (not used for auth) + createdAt: new Date(), + updatedAt: new Date(), + }, +}); +``` + +**Important Notes**: +- The Prisma User `id` field uses the **Keycloak user ID** (UUID) +- The `password` field in Prisma is not used for authentication (Keycloak handles that) +- Users must exist in Prisma before they can use Courrier + +### 3. Prisma Schema + +**User Model** (`prisma/schema.prisma`): +```prisma +model User { + id String @id @default(uuid()) + email String @unique + password String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + mailCredentials MailCredentials[] // One-to-many relationship + // ... other relations +} +``` + +**MailCredentials Model**: +```prisma +model MailCredentials { + id String @id @default(uuid()) + userId String + email String + password String? // Optional (for OAuth accounts) + host String + port Int + secure Boolean @default(true) + use_oauth Boolean @default(false) + refresh_token String? + access_token String? + token_expiry DateTime? + smtp_host String? + smtp_port Int? + smtp_secure Boolean? @default(false) + display_name String? + color String? @default("#0082c9") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, email]) // One email account per user + @@index([userId]) +} +``` + +## Courrier's Role: Adding Email Accounts + +### How Courrier Adds Email Accounts + +**Location**: `app/api/courrier/account/route.ts` (POST method) + +**Process**: +1. **Authentication Check**: Verifies user session exists +2. **User Existence Check**: Verifies user exists in Prisma database +3. **Connection Test**: Tests IMAP connection before saving +4. **Save Credentials**: Creates/updates `MailCredentials` record + +**Key Code Flow**: + +```typescript +// 1. Check if user exists in database +const userExistsInDB = await userExists(session.user.id); +if (!userExistsInDB) { + return NextResponse.json({ + error: 'User not found in database', + details: `The user ID from your session (${session.user.id}) doesn't exist in the database.` + }, { status: 400 }); +} + +// 2. Test connection +const testResult = await testEmailConnection(credentials); +if (!testResult.imap) { + return NextResponse.json({ + error: `Connection test failed: ${testResult.error}` + }, { status: 400 }); +} + +// 3. Save credentials +await saveUserEmailCredentials(session.user.id, email, credentials); +``` + +### Saving Email Credentials + +**Location**: `lib/services/email-service.ts` → `saveUserEmailCredentials()` + +**Process**: +1. Prepares database credentials object (excluding OAuth tokens) +2. Uses `upsert` to create or update `MailCredentials` +3. Caches full credentials (including OAuth tokens) in Redis + +**Key Code**: +```typescript +// Save to database using upsert +await prisma.mailCredentials.upsert({ + where: { + // Finds existing record by userId + email + userId_email: { + userId: userId, + email: credentials.email + } + }, + update: dbCredentials, + create: { + userId, + ...dbCredentials + } +}); + +// Cache full credentials (including OAuth) in Redis +await cacheEmailCredentials(userId, accountId, fullCreds); +``` + +**Important Notes**: +- OAuth tokens (access_token, refresh_token) are stored in **Redis only**, not in Prisma +- The Prisma `MailCredentials` table stores IMAP/SMTP settings +- The `password` field is optional (for OAuth accounts like Microsoft) + +### Microsoft OAuth Flow + +**Location**: `app/api/courrier/microsoft/callback/route.ts` + +For Microsoft accounts, the flow is: +1. User authorizes via Microsoft OAuth +2. Access token and refresh token are obtained +3. Credentials are saved with `use_oauth: true` +4. OAuth tokens are cached in Redis (not in Prisma) + +## Data Flow Diagram + +``` +┌─────────────┐ +│ Keycloak │ ← Primary user creation +└──────┬──────┘ + │ + │ Sync + ↓ +┌─────────────┐ +│ Prisma │ ← User record created +│ User │ +└──────┬──────┘ + │ + │ User adds email account + ↓ +┌─────────────┐ +│ Prisma │ ← MailCredentials created +│MailCredentials│ +└──────┬──────┘ + │ + │ OAuth tokens (if applicable) + ↓ +┌─────────────┐ +│ Redis │ ← OAuth tokens cached +└─────────────┘ +``` + +## Key Files Reference + +### User Creation +- `app/api/users/route.ts` - Creates users in Keycloak +- `scripts/sync-users.ts` - Syncs users from Keycloak to Prisma +- `app/api/sync-users/route.ts` - API endpoint for syncing users + +### Courrier Email Management +- `app/api/courrier/account/route.ts` - Add/update/delete email accounts +- `lib/services/email-service.ts` - Core email service functions + - `saveUserEmailCredentials()` - Saves email credentials to Prisma + - `getUserEmailCredentials()` - Retrieves credentials from Prisma + - `testEmailConnection()` - Tests IMAP/SMTP connection + +### Database Schema +- `prisma/schema.prisma` - Prisma schema definitions +- `lib/prisma.ts` - Prisma client instance + +### Authentication +- `app/api/auth/options.ts` - NextAuth configuration +- `lib/auth.ts` - Authentication helpers + +## Auto-Creation of Users + +**As of recent updates**, Courrier now automatically creates User records in Prisma if they don't exist when: +- Adding an email account (`/api/courrier/account` POST) +- Checking session status (`/api/courrier/session` GET) + +This handles cases where: +- The database was reset/lost but users still exist in Keycloak +- Users were created in Keycloak but never synced to Prisma + +The auto-creation uses session data from Keycloak to populate: +- `id`: Keycloak user ID (UUID) +- `email`: User's email from session +- `password`: Temporary random password (not used for auth, Keycloak handles authentication) + +## Common Issues & Solutions + +### Issue: "User not found in database" when adding email account + +**Cause**: User exists in Keycloak but not in Prisma database + +**Solution**: +- **Automatic**: The system now auto-creates users when needed +- **Manual**: Run the sync script to create users in Prisma: +```bash +npm run sync-users +# or +node scripts/sync-users.js +``` + +### Issue: Email credentials not saving + +**Check**: +1. User exists in Prisma: `prisma.user.findUnique({ where: { id: userId } })` +2. Connection test passes before saving +3. Unique constraint `[userId, email]` is not violated + +### Issue: OAuth tokens not persisting + +**Note**: OAuth tokens are stored in Redis, not Prisma. Check: +- Redis connection and TTL settings +- Redis cache functions in `lib/redis.ts` + +## Summary + +1. **Users are created in Keycloak first** (via `app/api/users/route.ts`) +2. **Users are synced to Prisma** (via sync scripts or API) +3. **Courrier adds email accounts** by creating `MailCredentials` records linked to existing Users +4. **OAuth tokens are cached in Redis**, not stored in Prisma +5. **Users must exist in Prisma** before they can add email accounts via Courrier + +Courrier is a **credentials management system** for existing users, not a user creation system. + diff --git a/app/api/courrier/account/route.ts b/app/api/courrier/account/route.ts index 181d5752..0731c475 100644 --- a/app/api/courrier/account/route.ts +++ b/app/api/courrier/account/route.ts @@ -4,6 +4,7 @@ import { authOptions } from "@/app/api/auth/options"; import { saveUserEmailCredentials, testEmailConnection } from '@/lib/services/email-service'; import { invalidateFolderCache } from '@/lib/redis'; import { prisma } from '@/lib/prisma'; +import bcrypt from 'bcryptjs'; // Define EmailCredentials interface inline since we're having import issues interface EmailCredentials { @@ -35,6 +36,57 @@ async function userExists(userId: string): Promise { } } +/** + * Ensure user exists in database, creating if missing + * Uses session data from Keycloak to populate user record + */ +async function ensureUserExists(session: any): Promise { + const userId = session.user.id; + const userEmail = session.user.email; + + if (!userId || !userEmail) { + throw new Error('Missing required user data in session'); + } + + try { + // Check if user exists + const existingUser = await prisma.user.findUnique({ + where: { id: userId } + }); + + if (existingUser) { + console.log(`User ${userId} already exists in database`); + return; + } + + // User doesn't exist, create it + console.log(`User ${userId} not found in database, creating from session data...`); + + // Generate a temporary random password (not used for auth, Keycloak handles that) + const tempPassword = await bcrypt.hash(Math.random().toString(36).slice(-10), 10); + + await prisma.user.create({ + data: { + id: userId, // Use Keycloak user ID + email: userEmail, + password: tempPassword, // Temporary password (Keycloak handles authentication) + createdAt: new Date(), + updatedAt: new Date(), + } + }); + + console.log(`Successfully created user ${userId} (${userEmail}) in database`); + } catch (error) { + console.error(`Error ensuring user exists:`, error); + // If it's a unique constraint error, user might have been created by another request + if (error instanceof Error && error.message.includes('Unique constraint')) { + console.log('User may have been created by concurrent request, continuing...'); + return; + } + throw error; + } +} + export async function POST(request: Request) { try { // Authenticate user @@ -46,16 +98,18 @@ export async function POST(request: Request) { ); } - // Verify that the user exists in the database - const userExistsInDB = await userExists(session.user.id); - if (!userExistsInDB) { - console.error(`User with ID ${session.user.id} not found in database`); + // Ensure user exists in database (create if missing) + // This handles cases where the database was reset but users still exist in Keycloak + try { + await ensureUserExists(session); + } catch (error) { + console.error(`Error ensuring user exists:`, error); return NextResponse.json( { - error: 'User not found in database', - details: `The user ID from your session (${session.user.id}) doesn't exist in the database. This may be due to a session/database mismatch.` + error: 'Failed to ensure user exists in database', + details: error instanceof Error ? error.message : 'Unknown error' }, - { status: 400 } + { status: 500 } ); } diff --git a/app/api/courrier/session/route.ts b/app/api/courrier/session/route.ts index 9485eec9..1bd62c5b 100644 --- a/app/api/courrier/session/route.ts +++ b/app/api/courrier/session/route.ts @@ -5,6 +5,7 @@ import { getMailboxes } from '@/lib/services/email-service'; import { getRedisClient } from '@/lib/redis'; import { getImapConnection } from '@/lib/services/email-service'; import { prisma } from '@/lib/prisma'; +import bcrypt from 'bcryptjs'; // Define extended MailCredentials type interface MailCredentials { @@ -34,6 +35,56 @@ const FOLDERS_CACHE_TTL = 3600; // 1 hour // Redis key for folders cache const FOLDERS_CACHE_KEY = (userId: string, accountId: string) => `email:folders:${userId}:${accountId}`; +/** + * Ensure user exists in database, creating if missing + * Uses session data from Keycloak to populate user record + */ +async function ensureUserExists(session: any): Promise { + const userId = session.user.id; + const userEmail = session.user.email; + + if (!userId || !userEmail) { + throw new Error('Missing required user data in session'); + } + + try { + // Check if user exists + const existingUser = await prisma.user.findUnique({ + where: { id: userId } + }); + + if (existingUser) { + return; + } + + // User doesn't exist, create it + console.log(`User ${userId} not found in database, creating from session data...`); + + // Generate a temporary random password (not used for auth, Keycloak handles that) + const tempPassword = await bcrypt.hash(Math.random().toString(36).slice(-10), 10); + + await prisma.user.create({ + data: { + id: userId, // Use Keycloak user ID + email: userEmail, + password: tempPassword, // Temporary password (Keycloak handles authentication) + createdAt: new Date(), + updatedAt: new Date(), + } + }); + + console.log(`Successfully created user ${userId} (${userEmail}) in database`); + } catch (error) { + console.error(`Error ensuring user exists:`, error); + // If it's a unique constraint error, user might have been created by another request + if (error instanceof Error && error.message.includes('Unique constraint')) { + console.log('User may have been created by concurrent request, continuing...'); + return; + } + throw error; + } +} + /** * This endpoint is called when the app initializes to check if the user has email credentials * and to start prefetching email data in the background if they do @@ -81,6 +132,19 @@ export async function GET() { name: session.user.name }); + // Ensure user exists in database (create if missing) + try { + await ensureUserExists(session); + } catch (error) { + console.error(`Error ensuring user exists:`, error); + return NextResponse.json({ + authenticated: true, + hasEmailCredentials: false, + error: 'Failed to ensure user exists in database', + details: error instanceof Error ? error.message : 'Unknown error' + }); + } + // Get user with their accounts console.log('Fetching user with ID:', session.user.id); const user = await prisma.user.findUnique({ @@ -89,7 +153,7 @@ export async function GET() { }); if (!user) { - console.error('User not found in database'); + console.error('User not found in database after creation attempt'); return NextResponse.json({ authenticated: true, hasEmailCredentials: false,