/** * Manual bank-statement FILE import (CAMT.052/053 from the online-banking export) * straight into iDempiere — for the period BEFORE EBICS was activated. * * Nuxt-only by design: no Laravel staging involved. Statements are created * Drafted with the same header/line shape the Laravel EBICS import writes * (`IdempiereBankStatementRepository::buildStatementPayload/buildLinePayload`), * so they show up in step ② "Abgleichen" like any other statement. * * DUPLICATE PROTECTION (the important part): * - identity of a booking = the bank's `AcctSvcrRef`, which the EBICS import * already stores as `C_BankStatementLine.EftTrxID`. An entry whose reference * exists on ANY non-voided statement of the bank account is never written. * - entries/lines without a reference fall back to date + amount + normalised * memo, counted as a multiset (two identical bookings on one day stay two). * - a day that already has a FOREIGN statement (EBICS / manual) is never * touched, even if the file knows more lines (`conflict`, shown as warning). * - a day whose statement was created by THIS importer and is still Drafted is * resumed (`resume`): only the missing lines are added — so a half-failed * import is repaired by simply uploading the same file again. * - the plan is recomputed against live iDempiere data right before writing. */ import { string } from 'alga-js' import { promises as fs } from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { fetchLinesForStatements } from '../bankReconciliation' import { buildDailyStatements, centsToAmount } from './camt052Parser' import type { CamtAccount, CamtEntry, CamtParseResult } from './camt052Parser' export const IMPORT_REF_PREFIX = 'camt052-' export const importRef = (iban: string, day: string): string => `${IMPORT_REF_PREFIX}${iban}-${day}`.slice(0, 60) const STORE_DIR = path.join(process.cwd(), 'data', 'bank-file-import') const TOKEN_TTL_MS = 24 * 60 * 60 * 1000 const CONTEXT_DAYS = 14 const shiftDay = (day: string, delta: number): string => { const date = new Date(day + 'T00:00:00Z') date.setUTCDate(date.getUTCDate() + delta) return date.toISOString().slice(0, 10) } const cut = (value: any, max: number): string => Array.from(String(value ?? '')).slice(0, max).join('') const normIban = (value: any): string => String(value || '').replace(/\s+/g, '').toUpperCase() const normMemo = (value: any): string => String(value || '').toUpperCase().replace(/[^A-Z0-9]/g, '').slice(0, 80) const fallbackKey = (day: string, cents: number, memo: any): string => `${day}|${cents}|${normMemo(memo)}` // --------------------------------------------------------------------------- // Upload store (disk — PM2 runs several instances, memory would not be shared) export const isValidToken = (token: any): boolean => /^[a-f0-9]{32}$/.test(String(token || '')) const purgeExpired = async () => { try { const names = await fs.readdir(STORE_DIR) for (const name of names) { if (!isValidToken(name)) continue const stat = await fs.stat(path.join(STORE_DIR, name)) if (Date.now() - stat.mtimeMs > TOKEN_TTL_MS) await fs.rm(path.join(STORE_DIR, name), { recursive: true, force: true }) } } catch {} } export const saveUpload = async (parsed: CamtParseResult, uploads: { name: string; data: Uint8Array }[]): Promise => { await purgeExpired() const token = crypto.randomBytes(16).toString('hex') const dir = path.join(STORE_DIR, token) await fs.mkdir(path.join(dir, 'raw'), { recursive: true }) await fs.writeFile(path.join(dir, 'parsed.json'), JSON.stringify({ createdAt: new Date().toISOString(), parsed })) for (let i = 0; i < uploads.length; i++) { await fs.writeFile(path.join(dir, 'raw', `${i + 1}-${path.basename(uploads[i].name)}`), uploads[i].data) } return token } export const loadUpload = async (token: string): Promise => { if (!isValidToken(token)) return null try { const raw = await fs.readFile(path.join(STORE_DIR, token, 'parsed.json'), 'utf-8') return JSON.parse(raw)?.parsed || null } catch { return null } } /** Keep the original bank files of an upload that actually wrote something. */ export const archiveUpload = async (token: string, iban: string) => { if (!isValidToken(token)) return try { const dir = path.join(STORE_DIR, token) try { await fs.access(path.join(dir, 'archived')); return } catch {} const target = path.join(STORE_DIR, 'archive', normIban(iban) || 'unknown') await fs.mkdir(target, { recursive: true }) const stamp = new Date().toISOString().replace(/[:.]/g, '-') for (const name of await fs.readdir(path.join(dir, 'raw'))) { await fs.copyFile(path.join(dir, 'raw', name), path.join(target, `${stamp}-${name}`)) } await fs.writeFile(path.join(dir, 'archived'), stamp) } catch (err: any) { console.warn('[BankFileImport] archive failed:', err?.message || err) } } // --------------------------------------------------------------------------- // iDempiere reads export interface ResolvedBankAccount { id: number; name: string; iban: string; orgId: number; orgName: string; bankName: string; currency: string; resolvedBy: 'iban' | 'alias' | 'manual' } const presentBankAccount = (acc: any, resolvedBy: ResolvedBankAccount['resolvedBy']): ResolvedBankAccount => ({ id: Number(acc.id), name: acc.Name || acc.AccountNo || String(acc.id), iban: normIban(acc.IBAN), orgId: Number(acc.AD_Org_ID?.id || 0), orgName: acc.AD_Org_ID?.identifier || '', bankName: acc.C_Bank_ID?.identifier || '', currency: acc.C_Currency_ID?.identifier || '', resolvedBy }) /** All active C_BankAccounts (the manual target picker in the preview). */ export const listBankAccounts = async (event: any, token: any): Promise => { const res: any = await event.context.fetch( `models/c_bankaccount?$filter=${string.urlEncode('IsActive eq true')}` + `&$select=Name,AccountNo,IBAN,AD_Org_ID,C_Bank_ID,C_Currency_ID&$orderby=Name asc&$top=200`, 'GET', token, null ) return (res?.records || []).map((acc: any) => presentBankAccount(acc, 'manual')) } // --------------------------------------------------------------------------- // IBAN → C_BankAccount assignments the user made in the preview ("this file // belongs to that account"). Needed when the bank changed the IBAN (merger) // but iDempiere still holds the old one. Persisted on disk next to the uploads // (data/ is kept across deploys) so the next upload resolves automatically. const ALIAS_FILE = path.join(STORE_DIR, 'account-aliases.json') const loadAliases = async (): Promise> => { try { const raw = JSON.parse(await fs.readFile(ALIAS_FILE, 'utf-8')) return raw && typeof raw === 'object' ? raw : {} } catch { return {} } } export const rememberBankAccountAlias = async (iban: string, bankAccountId: number) => { const key = normIban(iban) if (!key || !bankAccountId) return try { const aliases = await loadAliases() if (aliases[key] === bankAccountId) return aliases[key] = bankAccountId await fs.mkdir(STORE_DIR, { recursive: true }) await fs.writeFile(ALIAS_FILE, JSON.stringify(aliases, null, 2)) } catch (err: any) { console.warn('[BankFileImport] alias save failed:', err?.message || err) } } /** * C_BankAccount for a file IBAN: explicit `bankAccountId` (user picked it in * the preview) → exact IBAN match (compared normalised, iDempiere may store it * spaced) → remembered alias. Only active accounts qualify. */ export const resolveBankAccount = async (event: any, token: any, iban: string, bankAccountId: number | null = null): Promise => { const accounts = await listBankAccounts(event, token) if (bankAccountId) { const picked = accounts.find((acc) => acc.id === Number(bankAccountId)) return picked ? { ...picked, resolvedBy: 'manual' } : null } const wanted = normIban(iban) if (!wanted) return null const byIban = accounts.find((acc) => acc.iban === wanted) if (byIban) return { ...byIban, resolvedBy: 'iban' } const aliasId = (await loadAliases())[wanted] const byAlias = aliasId ? accounts.find((acc) => acc.id === Number(aliasId)) : null return byAlias ? { ...byAlias, resolvedBy: 'alias' } : null } interface ExistingStatement { id: number documentNo: string date: string docStatus: string eftReference: string beginningCents: number endingCents: number own: boolean maxLine: number lineCount: number } interface ExistingData { statements: ExistingStatement[] refs: Set unreferenced: Map } const loadExisting = async (event: any, token: any, bankAccountId: number, fromDay: string, toDay: string): Promise => { const filter = `C_BankAccount_ID eq ${bankAccountId} and StatementDate ge ${fromDay} and StatementDate le ${toDay}` const statements: ExistingStatement[] = [] const PAGE = 500 for (let skip = 0; skip < 5000; skip += PAGE) { const res: any = await event.context.fetch( `models/c_bankstatement?$filter=${string.urlEncode(filter)}` + `&$select=DocumentNo,StatementDate,BeginningBalance,EndingBalance,DocStatus,EftStatementReference` + `&$orderby=${string.urlEncode('StatementDate asc,C_BankStatement_ID asc')}&$top=${PAGE}&$skip=${skip}`, 'GET', token, null ) const records = res?.records || [] for (const rec of records) { const docStatus = rec.DocStatus?.id || rec.DocStatus || '' // Voided / reversed statements do not count as "already there". if (docStatus === 'VO' || docStatus === 'RE') continue const eftReference = String(rec.EftStatementReference || '') statements.push({ id: Number(rec.id), documentNo: rec.DocumentNo || '', date: String(rec.StatementDate || '').slice(0, 10), docStatus, eftReference, beginningCents: Math.round(Number(rec.BeginningBalance || 0) * 100), endingCents: Math.round(Number(rec.EndingBalance || 0) * 100), own: eftReference.startsWith(IMPORT_REF_PREFIX), maxLine: 0, lineCount: 0 }) } if (records.length < PAGE) break } const refs = new Set() const unreferenced = new Map() if (statements.length) { const byId = new Map(statements.map((s) => [s.id, s])) const lines = await fetchLinesForStatements( event, token, statements.map((s) => s.id), '$select=C_BankStatement_ID,Line,StatementLineDate,StmtAmt,EftTrxID,EftMemo', 50000 ) for (const line of lines) { const statement = byId.get(Number(line.C_BankStatement_ID?.id)) if (statement) { statement.lineCount++ statement.maxLine = Math.max(statement.maxLine, Number(line.Line || 0)) } const ref = String(line.EftTrxID || '').trim() if (ref) { refs.add(ref) } else { const key = fallbackKey(String(line.StatementLineDate || '').slice(0, 10), Math.round(Number(line.StmtAmt || 0) * 100), line.EftMemo) unreferenced.set(key, (unreferenced.get(key) || 0) + 1) } } } return { statements, refs, unreferenced } } // --------------------------------------------------------------------------- // Plan export type DayVerdict = 'would_create' | 'resume' | 'already_in_idempiere' | 'conflict' | 'no_lines' export type LineVerdict = 'would_create' | 'already_in_idempiere' | 'skip_non_eur' export interface PlannedLine { verdict: LineVerdict entry: CamtEntry } export interface PlannedDay { date: string verdict: DayVerdict openingCents: number | null closingCents: number | null netCents: number lines: PlannedLine[] existing: ExistingStatement[] counts: { wouldCreate: number; known: number; nonEur: number } } export interface AccountPlan { iban: string ownerName: string bankName: string bankAccount: ResolvedBankAccount | null days: PlannedDay[] balanceCheck: ReturnType['check'] chain: { ok: boolean; checks: { kind: string; day: string; statementDate: string; documentNo: string; expectedCents: number; computedCents: number; diffCents: number }[] } skipped: CamtAccount['skipped'] } /** * Plan one account against LIVE iDempiere data. `onlyDays` restricts the * evaluation (import chunks) — balances still come from the whole file. */ export const planAccount = async (event: any, token: any, account: CamtAccount, onlyDays: string[] | null = null, bankAccountId: number | null = null): Promise => { const bankAccount = await resolveBankAccount(event, token, account.iban, bankAccountId) let { days, check } = buildDailyStatements(account) const wanted = onlyDays ? new Set(onlyDays) : null const scopeDays = days.filter((d) => !wanted || wanted.has(d.date)) const plan: AccountPlan = { iban: account.iban, ownerName: account.ownerName, bankName: account.bankName, bankAccount, days: [], balanceCheck: check, chain: { ok: true, checks: [] }, skipped: account.skipped } if (!bankAccount || !scopeDays.length) { plan.days = scopeDays.map((d) => ({ date: d.date, verdict: 'no_lines' as DayVerdict, openingCents: d.openingCents, closingCents: d.closingCents, netCents: d.netCents, lines: d.entries.map((entry) => ({ verdict: 'would_create' as LineVerdict, entry })), existing: [], counts: { wouldCreate: 0, known: 0, nonEur: 0 } })) return plan } const fromDay = shiftDay(scopeDays[0].date, -CONTEXT_DAYS) const toDay = shiftDay(scopeDays[scopeDays.length - 1].date, CONTEXT_DAYS) const existing = await loadExisting(event, token, bankAccount.id, fromDay, toDay) // Files without any balance: anchor on the closing of the statement before. if (!check.anchored && days.length) { const before = existing.statements.filter((s) => s.date < days[0].date).pop() if (before) ({ days, check } = buildDailyStatements(account, before.endingCents)) plan.balanceCheck = check } const dayByDate = new Map(days.map((d) => [d.date, d])) // Multiset of unreferenced existing lines — consumed as entries match. const unreferenced = new Map(existing.unreferenced) for (const scoped of scopeDays) { const day = dayByDate.get(scoped.date)! const sameDay = existing.statements.filter((s) => s.date === day.date) const lines: PlannedLine[] = [] const counts = { wouldCreate: 0, known: 0, nonEur: 0 } for (const entry of day.entries) { let verdict: LineVerdict = 'would_create' if (entry.currency !== 'EUR') { verdict = 'skip_non_eur' } else if (entry.bankReference && existing.refs.has(cut(entry.bankReference, 40))) { verdict = 'already_in_idempiere' } else { const key = fallbackKey(entry.bookingDate, entry.amountCents, entry.remittanceInfo) const left = unreferenced.get(key) || 0 if (left > 0) { unreferenced.set(key, left - 1) verdict = 'already_in_idempiere' } } if (verdict === 'would_create') counts.wouldCreate++ else if (verdict === 'skip_non_eur') counts.nonEur++ else counts.known++ lines.push({ verdict, entry }) } let verdict: DayVerdict if (counts.wouldCreate === 0) verdict = counts.known ? 'already_in_idempiere' : 'no_lines' else if (!sameDay.length) verdict = 'would_create' else if (sameDay.length === 1 && sameDay[0].own && sameDay[0].docStatus === 'DR' && sameDay[0].eftReference === importRef(account.iban, day.date)) verdict = 'resume' else verdict = 'conflict' plan.days.push({ date: day.date, verdict, openingCents: day.openingCents, closingCents: day.closingCents, netCents: day.netCents, lines, existing: sameDay, counts }) } // Balance chain vs. what iDempiere already holds (informational, never blocks): // a) same-day statements must show the same opening/closing as the file // b) the first existing statement after a NEW day must open with that day's closing const addCheck = (kind: string, day: string, statement: ExistingStatement, expectedCents: number, computedCents: number | null) => { if (computedCents === null) return const diffCents = computedCents - expectedCents if (diffCents !== 0) plan.chain.ok = false plan.chain.checks.push({ kind, day, statementDate: statement.date, documentNo: statement.documentNo, expectedCents, computedCents, diffCents }) } for (const planned of plan.days) { if (planned.existing.length === 1 && planned.verdict === 'already_in_idempiere' && !planned.counts.nonEur) { // Reported only on a mismatch (one row per differing balance). const statement = planned.existing[0] if (statement.beginningCents !== planned.openingCents) addCheck('same_day_opening', planned.date, statement, statement.beginningCents, planned.openingCents) if (statement.endingCents !== planned.closingCents) addCheck('same_day_closing', planned.date, statement, statement.endingCents, planned.closingCents) } } const newDays = plan.days.filter((d) => d.verdict === 'would_create' || d.verdict === 'resume') for (const planned of newDays) { const nextFileDay = days.find((d) => d.date > planned.date) const nextStatement = existing.statements.find((s) => s.date > planned.date) // Only where the existing statement is the DIRECT successor of this new day. if (nextStatement && (!nextFileDay || nextStatement.date <= nextFileDay.date)) { addCheck('next_opening', planned.date, nextStatement, nextStatement.beginningCents, planned.closingCents) } } return plan } /** API shape of a plan (amounts as numbers, entries flattened). */ export const presentPlan = (plan: AccountPlan) => { const presentExisting = (s: ExistingStatement) => ({ id: s.id, documentNo: s.documentNo, docStatus: s.docStatus, own: s.own, lineCount: s.lineCount }) const days = plan.days.map((d) => ({ date: d.date, verdict: plan.bankAccount ? d.verdict : 'bank_account_missing', opening: centsToAmount(d.openingCents), closing: centsToAmount(d.closingCents), net: centsToAmount(d.netCents), counts: d.counts, existing: d.existing.map(presentExisting), lines: d.lines.map((l) => ({ verdict: l.verdict, bookingDate: l.entry.bookingDate, valueDate: l.entry.valueDate, amount: l.entry.amount, currency: l.entry.currency, counterpart: l.entry.counterpartName || '', counterpartIban: l.entry.counterpartIban || '', remittanceInfo: l.entry.remittanceInfo || '', bankReference: l.entry.bankReference || '', batchCount: l.entry.batchCount })) })) const count = (verdict: string) => days.filter((d) => d.verdict === verdict).length const sum = (pick: (d: PlannedDay) => number) => plan.days.reduce((total, d) => total + pick(d), 0) const writable = plan.bankAccount ? plan.days.filter((d) => d.verdict === 'would_create' || d.verdict === 'resume') : [] return { iban: plan.iban, ownerName: plan.ownerName, bankName: plan.bankName, bankAccount: plan.bankAccount, period: { from: days[0]?.date || null, to: days[days.length - 1]?.date || null }, totals: { days: days.length, daysWouldCreate: count('would_create'), daysResume: count('resume'), daysKnown: count('already_in_idempiere'), daysConflict: count('conflict'), lines: sum((d) => d.lines.length), linesWouldCreate: writable.reduce((total, d) => total + d.counts.wouldCreate, 0), linesKnown: sum((d) => d.counts.known), linesNonEur: sum((d) => d.counts.nonEur), linesConflict: plan.days.filter((d) => d.verdict === 'conflict').reduce((total, d) => total + d.counts.wouldCreate, 0), pendingSkipped: plan.skipped.pending, duplicateInUpload: plan.skipped.duplicateInUpload }, balanceCheck: { ok: plan.balanceCheck.ok, anchored: plan.balanceCheck.anchored, anchor: plan.balanceCheck.anchor ? { ...plan.balanceCheck.anchor, amount: centsToAmount(plan.balanceCheck.anchor.amountCents) } : null, diffs: plan.balanceCheck.diffs.map((d) => ({ type: d.type, date: d.date, file: d.file, expected: centsToAmount(d.expectedCents), computed: centsToAmount(d.computedCents), diff: centsToAmount(d.diffCents) })) }, chain: { ok: plan.chain.ok, checks: plan.chain.checks.map((c) => ({ kind: c.kind, day: c.day, statementDate: c.statementDate, documentNo: c.documentNo, expected: centsToAmount(c.expectedCents), computed: centsToAmount(c.computedCents), diff: centsToAmount(c.diffCents) })) }, writableDays: writable.map((d) => d.date), days } } // --------------------------------------------------------------------------- // Write const buildLinePayload = (entry: CamtEntry, statementId: number, orgId: number, lineNo: number) => { const description = [entry.counterpartName, entry.remittanceInfo].filter(Boolean).join(' | ') const payload: any = { AD_Org_ID: { id: orgId, tableName: 'AD_Org' }, C_BankStatement_ID: { id: statementId, tableName: 'C_BankStatement' }, line: lineNo, statementLineDate: entry.bookingDate, dateAcct: entry.bookingDate, stmtAmt: entry.amount, trxAmt: entry.amount, description: cut(description || entry.additionalInfo || 'Bank transaction', 255), eftPayee: cut(entry.counterpartName, 255), eftPayeeAccount: cut(entry.counterpartIban, 40), eftMemo: cut(entry.remittanceInfo, 2000) } if (entry.valueDate) payload.valutaDate = entry.valueDate const reference = entry.endToEndId || entry.bankReference if (reference) payload.referenceNo = cut(reference, 40) // The duplicate-detection key — same column the EBICS import fills. if (entry.bankReference) payload.eftTrxID = cut(entry.bankReference, 40) return payload } const isAuthError = (err: any): boolean => [401, 403].includes(Number(err?.statusCode ?? err?.status ?? err?.response?.status)) const errorText = (err: any): string => String(err?.data?.detail || err?.data?.title || err?.data?.message || err?.message || err).slice(0, 300) export interface DayResult { date: string status: 'created' | 'resumed' | 'skipped' | 'failed' verdict: string statementId: number | null documentNo: string linesCreated: number linesSkipped: number errors: string[] } /** * Write ONE planned day. A failing line never rolls the day back — the * statement stays Drafted and a re-upload resumes it. Auth errors are re-thrown * so the route's refresh-retry can re-plan (idempotent thanks to `resume`). */ export const writeDay = async (event: any, token: any, plan: AccountPlan, day: PlannedDay, sourceLabel: string): Promise => { const result: DayResult = { date: day.date, status: 'skipped', verdict: day.verdict, statementId: null, documentNo: '', linesCreated: 0, linesSkipped: day.counts.known + day.counts.nonEur, errors: [] } const bankAccount = plan.bankAccount if (!bankAccount || (day.verdict !== 'would_create' && day.verdict !== 'resume')) return result const organizationId = bankAccount.orgId || Number(getCookie(event, 'logship_organization_id') || 0) let statementId: number let lineNo: number if (day.verdict === 'resume') { statementId = day.existing[0].id result.documentNo = day.existing[0].documentNo lineNo = Math.floor(day.existing[0].maxLine / 10) * 10 + 10 result.status = 'resumed' } else { try { const header: any = await event.context.fetch('models/c_bankstatement', 'POST', token, { AD_Org_ID: { id: organizationId, tableName: 'AD_Org' }, C_BankAccount_ID: { id: bankAccount.id, tableName: 'C_BankAccount' }, name: cut(`${bankAccount.name} ${day.date}`, 60), statementDate: day.date, // dateacct is NOT NULL and its AD default is not applied on REST POSTs. dateAcct: day.date, beginningBalance: centsToAmount(day.openingCents) ?? 0, endingBalance: centsToAmount(day.closingCents) ?? 0, description: cut(`CAMT.052 Datei-Import (${sourceLabel})`, 255), eftStatementReference: importRef(plan.iban, day.date), isManual: false }) if (!header?.id) throw new Error('iDempiere lieferte keine Auszugs-ID') statementId = Number(header.id) result.documentNo = header.DocumentNo || '' lineNo = 10 result.status = 'created' } catch (err: any) { if (isAuthError(err)) throw err result.status = 'failed' result.errors.push('Auszug: ' + errorText(err)) return result } } result.statementId = statementId for (const line of day.lines) { if (line.verdict !== 'would_create') continue try { await event.context.fetch('models/c_bankstatementline', 'POST', token, buildLinePayload(line.entry, statementId, organizationId, lineNo)) result.linesCreated++ lineNo += 10 } catch (err: any) { if (isAuthError(err)) throw err result.errors.push(`${line.entry.amount} ${line.entry.counterpartName || ''}: ${errorText(err)}`.trim()) } } if (result.errors.length) result.status = 'failed' return result }