Jobs asynchrones IA avec Next.js 14 : file d’attente, webhooks et polling intelligent

Client (Next.js)
  │  POST /api/jobs  { type: "report", prompt, webhookUrl? }
  │  ← 202 { jobId, status: "queued", pollUrl: "/api/jobs/abc123" }
  │
  ├─ 1. API route Next.js : valide, déduplique (idempotency-key), enfile
  ├─ 2. File d'attente (Vercel Queue / Upstash QStash / BullMQ sur Redis)
  ├─ 3. Worker : consomme le job → appels LLM (Groq/Gemini/OpenAI) → stocke résultat (DB + S3/R2)
  └─ 4. Notification : polling côté client OU webhook sortant OU email

// src/app/api/jobs/route.ts
import { NextRequest } from 'next/server'
import { safeQuery } from '@/lib/db'

export async function POST(req: NextRequest) {
  const body = await req.json()
  const { type, prompt, webhookUrl } = body as { type: string; prompt: string; webhookUrl?: string }
  if (!prompt || typeof prompt !== 'string' || prompt.length < 3) {
    return new Response('Missing prompt', { status: 400 })
  }
  if (prompt.length > 20000) return new Response('Prompt too long', { status: 413 })

  // Idempotence : même Idempotency-Key → même jobId (évite double facturation au retry)
  const idempotencyKey = req.headers.get('idempotency-key') || null
  const jobId = idempotencyKey
    ? 'job_' + Buffer.from(idempotencyKey).toString('base64url').slice(0, 24)
    : 'job_' + Date.now().toString(36) + Math.random().toString(36).slice(2, 8)

  const existing = idempotencyKey
    ? await safeQuery(prisma => prisma.job.findUnique({ where: { id: jobId } }), null)
    : null
  if (existing) {
    return Response.json({ jobId, status: existing.status, pollUrl: `/api/jobs/${jobId}` }, { status: 200 })
  }

  await safeQuery(prisma => prisma.job.create({
    data: { id: jobId, type: type || 'generic', prompt: prompt.slice(0, 20000), status: 'queued', webhookUrl: webhookUrl || null }
  }), null)

  // Enfilement : Upstash QStash (HTTP) — remplacez par Vercel Queue si dispo
  const qstashUrl = process.env.QSTASH_URL // https://qstash.upstash.io/v2/publish/https://votre-app.vercel.app/api/jobs/worker
  if (qstashUrl && process.env.QSTASH_TOKEN) {
    await fetch(qstashUrl, {
      method: 'POST',
      headers: { Authorization: `Bearer ${process.env.QSTASH_TOKEN}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ jobId }),
      signal: AbortSignal.timeout(5000),
    }).catch(() => { /* queue retry côté QStash */ })
  }

  return Response.json({ jobId, status: 'queued', pollUrl: `/api/jobs/${jobId}` }, { status: 202 })
}

// src/app/api/jobs/worker/route.ts
import { NextRequest } from 'next/server'
import { safeQuery } from '@/lib/db'
import { callAI } from '@/lib/ai' // wrapper Groq → Gemini → OpenAI avec retry

export async function POST(req: NextRequest) {
  // Vérif QStash signature (ou Vercel Queue secret)
  const sig = req.headers.get('upstash-signature')
  if (process.env.QSTASH_CURRENT_SIGNING_KEY && sig) {
    // vérifiez la signature côté prod (lib @upstash/qstash)
  }

  const { jobId } = await req.json() as { jobId: string }
  if (!jobId) return new Response('Missing jobId', { status: 400 })

  await safeQuery(p => p.job.update({ where: { id: jobId }, data: { status: 'processing' } }), null)

  try {
    const job = await safeQuery(p => p.job.findUnique({ where: { id: jobId } }), null) as { prompt: string } | null
    if (!job) return new Response('Not found', { status: 404 })

    // Appel LLM long (peut durer 30-120 s) — hors timeout client
    const result = await callAI(job.prompt)

    await safeQuery(p => p.job.update({ where: { id: jobId }, data: { status: 'done', result } }), null)

    // Webhook sortant si demandé
    const full = await safeQuery(p => p.job.findUnique({ where: { id: jobId } }), null) as { webhookUrl?: string } | null
    if (full?.webhookUrl) {
      await fetch(full.webhookUrl, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ jobId, status: 'done', result }),
        signal: AbortSignal.timeout(8000),
      }).catch(() => {})
    }

    return Response.json({ ok: true })
  } catch (e) {
    await safeQuery(p => p.job.update({ where: { id: jobId }, data: { status: 'failed', result: String(e).slice(0, 2000) } }), null)
    return new Response('failed', { status: 500 }) // déclenche retry QStash
  }
}

// src/app/api/jobs/[id]/route.ts
import { NextRequest } from 'next/server'
import { safeQuery } from '@/lib/db'

export async function GET(_req: NextRequest, { params }: { params: { id: string } }) {
  const job = await safeQuery(p => p.job.findUnique({ where: { id: params.id } }), null) as
    | { id: string; status: string; result: string | null } | null
  if (!job) return new Response('Not found', { status: 404 })
  return Response.json({ jobId: job.id, status: job.status, result: job.result || undefined })
}

// Hook React : polling avec backoff
import { useEffect, useState } from 'react'

export function useJobPoll(jobId: string | null) {
  const [data, setData] = useState<{ status: string; result?: string } | null>(null)
  useEffect(() => {
    if (!jobId) return
    let delay = 1000, timer: ReturnType<typeof setTimeout>
    let stopped = false
    async function tick() {
      const res = await fetch(`/api/jobs/${jobId}`)
      if (!res.ok) { delay = Math.min(delay * 1.5, 10000); if (!stopped) timer = setTimeout(tick, delay); return }
      const json = await res.json()
      setData(json)
      if (json.status === 'queued' || json.status === 'processing') {
        delay = Math.min(delay * 1.5, 5000)
        if (!stopped) timer = setTimeout(tick, delay)
      }
    }
    tick()
    return () => { stopped = true; clearTimeout(timer) }
  }, [jobId])
  return data
}

Mode         Échecs timeout   Coût retry/mois   P95 perçu client
Synchrone    8–12 %           ~38 €               18 s (spinner bloqué)
Async+poll   < 0.5 %          ~2 €                0.2 s (ACK) + notif à la fin