#!/bin/bash set -euo pipefail # ----------------------------- # CONFIG # ----------------------------- NODE_VERSION="v22.22.3" NVM_DIR="/root/.nvm" APP_NAME="erpfrontend" # PM2 process name used in ecosystem.cjs MEDIASERVER_NAME="mediaserver" # PM2 process name for media server SRC_DIR="/root/test/workspace/erp-frontend-pipeline-production" STAGE_DIR="/opt/erp-nuxt-frontend-temp" LIVE_DIR="/opt/erp-nuxt-frontend" ROLLBACK_DIR="/opt/erp-nuxt-frontend-rollback" MEDIASERVER_DIR="/opt/mediaserver" # Every SQLite DB under data/ that must survive a deploy. data/*.db is gitignored, # so it is NOT in the build artifact — anything missing here is recreated empty # on the next deploy (this is what silently wiped chat.db before). # amazon-ads.db = Amazon invoices page: uploaded-but-not-imported PDFs + PDF cache index (booked # invoices are listed from iDempiere, so losing it only drops PENDING uploads — it was wiped by # every deploy until 2026-09-18 because it was missing here). DB_FILES=("push-subscriptions.db" "chat.db" "email-verify.db" "fcm-tokens.db" "legal-docs.db" "contract-signing.db" "amazon-ads.db" "settlement-exports.db") # Other frontend-owned runtime data under data/ that must survive too (all gitignored): # - contract-signing/ PDF packages of digital signing links (one dir per token) — # the signing portal serves these; losing them breaks open links # - ebay-signing-keys.json eBay digital-signature key pair # mobile-releases = LogShip Mobile APK channel (server/utils/mobileReleases.ts) — published at # runtime, NOT part of the build; dropping it here would wipe every APK on the next deploy. # - amazon-ads/ cached original PDFs of the Amazon invoices page (pairs with amazon-ads.db) # - lead-assessments/ KI-Analyse results + job state per lead (server/utils/leads/leadAssessment.ts) — # paid Claude calls; the lead page reopens the last assessment from here # - lead-chats/ lead chat history per lead (server/utils/leads/leadChat.ts) # - call-transcripts/ whisper transcript cache per recording (server/utils/telephony/recordings.ts) — # losing it means every recording is transcribed again (minutes of CPU each) # - bank-file-import/ CAMT.052 upload originals + parsed previews (server/utils/banking/bankFileImport.ts) # Whenever a new store under data/ is added to .gitignore, add it here too — otherwise it is # wiped on the next deploy. DATA_EXTRA=("contract-signing" "ebay-signing-keys.json" "mobile-releases" "amazon-ads" "lead-assessments" "lead-chats" "call-transcripts" "bank-file-import") DB_LIVE_DIR="$LIVE_DIR/data" DB_BACKUP_DIR="/opt/erp-nuxt-frontend-db-backups" ENV_SRC="$SRC_DIR/.env-prod" ENV_DST="$STAGE_DIR/.env" # ----------------------------- # NVM + NODE 22 # ----------------------------- export NVM_DIR="$NVM_DIR" # shellcheck disable=SC1090 [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" nvm install "$NODE_VERSION" nvm alias default "$NODE_VERSION" nvm use "$NODE_VERSION" # Ensure pm2 is installed for THIS Node version npm install -g pm2 NODE_BIN="$NVM_DIR/versions/node/$NODE_VERSION/bin/node" NPM_BIN="$NVM_DIR/versions/node/$NODE_VERSION/bin/npm" PM2_BIN="$NVM_DIR/versions/node/$NODE_VERSION/bin/pm2" echo "Using Node: $("$NODE_BIN" -v)" echo "ABI: $("$NODE_BIN" -p 'process.versions.modules')" echo "PM2: $("$PM2_BIN" -v)" # ----------------------------- # STAGE WORKSPACE # ----------------------------- mkdir -p "$STAGE_DIR" rm -rf "${STAGE_DIR:?}"/* cp -r "$SRC_DIR"/* "$STAGE_DIR"/ cp "$ENV_SRC" "$ENV_DST" # Inject build metadata cd "$SRC_DIR" git_hash=$(git rev-parse --short HEAD || echo "unknown") current_datetime=$(date '+%d.%m.%Y %H:%M') cd "$STAGE_DIR" sed -i "s//$git_hash/g" "$ENV_DST" sed -i "s//$current_datetime/g" "$ENV_DST" export NODE_OPTIONS="--max-old-space-size=12288" # Clean install + build under Node 22. # npm ci installs EXACTLY the committed package-lock.json — no re-resolution # against the live registry (a lockfile-less `npm install` here let a fresh # @vitejs/devtools publish crash npm's resolver on 2026-07-25). rm -rf node_modules .nuxt .output $NPM_BIN ci $NPM_BIN run build # ----------------------------- # VERSION SERVICE WORKER CACHE # ----------------------------- echo "Versioning service worker cache with git hash: $git_hash" # Match ANY current value: the literal 'logship-erp-v1' pattern used before was a silent no-op # from 2026-04-14 (file moved to 'logship-erp-v2') on, so clients never dropped the old cache. if [ -f "$STAGE_DIR/.output/public/sw-notifications.js" ]; then sed -i "s/const CACHE_NAME = '[^']*'/const CACHE_NAME = 'logship-erp-$git_hash'/" \ "$STAGE_DIR/.output/public/sw-notifications.js" if grep -q "const CACHE_NAME = 'logship-erp-$git_hash'" "$STAGE_DIR/.output/public/sw-notifications.js"; then echo "Service worker cache versioned: logship-erp-$git_hash" else echo "Warning: CACHE_NAME line not found in sw-notifications.js - cache NOT versioned" fi else echo "Warning: Service worker file not found at expected location" fi # ----------------------------- # REBUILD NATIVE MODULES FOR NODE 22 — IN THE STAGE DIR, BEFORE THE SWAP # (2026-10-30: these builds took minutes AFTER the live dir had been replaced while the old # pm2 process still served requests on deleted inodes — a contract-signing row written in that # window was lost. Everything slow happens here; the swap below is seconds.) # ----------------------------- # better-sqlite3 >= 13 ships prebuilds/linux-x64.node inside the package (Nitro copies the whole # prebuilds/ dir into .output/server/node_modules) and has no install script, so this rebuild is a # no-op safety net. Keep it: it still matters if the package ever falls back to node-gyp. cd "$STAGE_DIR/.output/server" echo "Rebuilding better-sqlite3 for Node $NODE_VERSION..." $NPM_BIN rebuild better-sqlite3 # sqlite3 (server/utils/sqliteHelper.ts): the npm prebuilt binary of sqlite3 >= 6 is linked # against glibc 2.38, the prod host (Debian 12) has glibc 2.36 -> "GLIBC_2.38 not found" at # startup, the server never binds :3001 and nginx answers 502 (outage 2026-10-22). # Compile it from source in the FULL node_modules (the Nitro-traced copy under .output/server # has no binding.gyp/deps — an `npm rebuild` there only deletes the binary) and copy the # result into the server bundle. Needs gcc/g++/make/python3 on the host (present). echo "Building sqlite3 from source for the host glibc..." cd "$STAGE_DIR" $NPM_BIN rebuild sqlite3 --build-from-source SQLITE3_BIN="$STAGE_DIR/node_modules/sqlite3/build/Release/node_sqlite3.node" if [ -f "$SQLITE3_BIN" ]; then mkdir -p "$STAGE_DIR/.output/server/node_modules/sqlite3/build/Release" cp -f "$SQLITE3_BIN" "$STAGE_DIR/.output/server/node_modules/sqlite3/build/Release/" echo " needs $(objdump -T "$SQLITE3_BIN" | grep -o 'GLIBC_[0-9.]*' | sort -Vu | tail -1), host has $(ldd --version | head -1 | grep -o '[0-9.]*$')" else echo "ERROR: sqlite3 source build produced no binary" >&2 exit 1 fi # Hard gate: every native module in the bundle must load under the deploy Node BEFORE pm2 is # touched — a broken binding is otherwise an async crash that pm2 still reports as "online". cd "$STAGE_DIR/.output/server" "$NODE_BIN" -e 'require("./node_modules/sqlite3"); require("./node_modules/better-sqlite3"); console.log(" verified: sqlite3 + better-sqlite3 load OK")' # ----------------------------- # COPY RUNTIME ASSETS NITRO DOES NOT BUNDLE # Nitro externalizes tesseract.js-core / pdfjs-dist but traces only their JS # entry — NOT tesseract's *.wasm or pdfjs' worker. The incoming-invoice OCR # pipeline (Eingangsrechnungen) needs them at runtime, so copy them from the # real node_modules into .output/server/node_modules. Idempotent; safe to re-run. # ----------------------------- echo "Copying OCR/PDF runtime assets into .output/server/node_modules..." SERVER_NM="$STAGE_DIR/.output/server/node_modules" SRC_NM="$STAGE_DIR/node_modules" # tesseract.js-core: the *.wasm core files (+ JS wrappers) for OCR (scanned PDFs) if [ -d "$SRC_NM/tesseract.js-core" ]; then mkdir -p "$SERVER_NM/tesseract.js-core" cp -rf "$SRC_NM/tesseract.js-core/." "$SERVER_NM/tesseract.js-core/" echo " ✓ tesseract.js-core (wasm)" else echo " ⚠ tesseract.js-core not found in node_modules" fi # pdfjs-dist: the Node 'fake worker' (pdf.worker.mjs) must sit beside pdf.mjs, # else Tier-1 text extraction throws and every PDF falls through to OCR. if [ -d "$SRC_NM/pdfjs-dist/legacy/build" ]; then mkdir -p "$SERVER_NM/pdfjs-dist/legacy/build" cp -f "$SRC_NM/pdfjs-dist/legacy/build/"pdf*.mjs "$SERVER_NM/pdfjs-dist/legacy/build/" 2>/dev/null || true echo " ✓ pdfjs-dist worker" else echo " ⚠ pdfjs-dist/legacy/build not found in node_modules" fi # canvas: native; ONLY the Tier-2 rasterize fallback used when pdftoppm (poppler) # is absent on the host. Preferred path is poppler-utils — install that instead # of relying on this copy. Copied best-effort only if present and not already there. if [ -d "$SRC_NM/canvas" ] && [ ! -e "$SERVER_NM/canvas/build/Release/canvas.node" ]; then mkdir -p "$SERVER_NM/canvas" cp -rf "$SRC_NM/canvas/." "$SERVER_NM/canvas/" 2>/dev/null || true echo " ✓ canvas (rasterize fallback)" fi # Sanity check (non-fatal): confirm the critical assets landed ls "$SERVER_NM/tesseract.js-core/"*.wasm >/dev/null 2>&1 && echo " verified: tesseract wasm present" || echo " ⚠ tesseract wasm still missing" [ -f "$SERVER_NM/pdfjs-dist/legacy/build/pdf.worker.mjs" ] && echo " verified: pdfjs worker present" || echo " ⚠ pdfjs worker still missing" # ----------------------------- # BACKUP SQLITE DBs # ----------------------------- echo "Backing up SQLite databases..." mkdir -p "$DB_BACKUP_DIR" backup_ts="$(date +%Y%m%d-%H%M%S)" for db in "${DB_FILES[@]}"; do if [ -f "$LIVE_DIR/data/$db" ]; then base="${db%.db}" # Online backup through SQLite (includes WAL frames not yet checkpointed); plain cp as fallback "$NODE_BIN" -e 'const D=require(process.argv[1]); new D(process.argv[2],{readonly:true}).backup(process.argv[3]).then(()=>process.exit(0)).catch(e=>{console.error(e.message);process.exit(1)})' \ "$STAGE_DIR/node_modules/better-sqlite3" "$LIVE_DIR/data/$db" "$DB_BACKUP_DIR/${base}-${backup_ts}.db" \ || cp "$LIVE_DIR/data/$db" "$DB_BACKUP_DIR/${base}-${backup_ts}.db" echo "Database backed up: $DB_BACKUP_DIR/${base}-${backup_ts}.db" # keep last 10 backups per database (cd "$DB_BACKUP_DIR" && ls -t "${base}-"*.db 2>/dev/null | tail -n +11 | xargs -r rm) else echo "No existing $db found - fresh deployment for it" fi done # ----------------------------- # DEPLOY (with rollback) # ----------------------------- # Stop the old server FIRST: its SQLite connections close (WAL checkpointed, files consistent) and # no request can write into the soon-deleted directory. Downtime = this swap only (seconds), since # the slow native builds already ran in the stage dir above. $PM2_BIN stop "$APP_NAME" || true rm -rf "$ROLLBACK_DIR" if [ -d "$LIVE_DIR" ]; then cp -rf "$LIVE_DIR" "$ROLLBACK_DIR" fi rm -rf "$LIVE_DIR" mkdir -p "$LIVE_DIR" cp -r "$STAGE_DIR"/. "$LIVE_DIR"/ # ----------------------------- # RESTORE SQLITE DBs # ----------------------------- echo "Restoring SQLite databases..." mkdir -p "$DB_LIVE_DIR" for db in "${DB_FILES[@]}"; do if [ -f "$ROLLBACK_DIR/data/$db" ]; then cp "$ROLLBACK_DIR/data/$db" "$DB_LIVE_DIR"/ # Carry over WAL sidecars too: chat.db runs in WAL mode, so the most recent # messages may still live in -wal and not yet be checkpointed into the .db. [ -f "$ROLLBACK_DIR/data/$db-wal" ] && cp "$ROLLBACK_DIR/data/$db-wal" "$DB_LIVE_DIR"/ || true [ -f "$ROLLBACK_DIR/data/$db-shm" ] && cp "$ROLLBACK_DIR/data/$db-shm" "$DB_LIVE_DIR"/ || true echo "Database $db restored from rollback" else echo "No previous $db found - will be created on first run" fi done # Extra data files/dirs (see DATA_EXTRA above) for item in "${DATA_EXTRA[@]}"; do if [ -e "$ROLLBACK_DIR/data/$item" ]; then cp -rf "$ROLLBACK_DIR/data/$item" "$DB_LIVE_DIR"/ echo "Data item $item restored from rollback" else echo "No previous data/$item found - will be created on first use" fi done chown -R root:root "$DB_LIVE_DIR" chmod 755 "$DB_LIVE_DIR" for db in "${DB_FILES[@]}"; do for f in "$db" "$db-wal" "$db-shm"; do [ -f "$DB_LIVE_DIR/$f" ] && chmod 644 "$DB_LIVE_DIR/$f" || true done done # ----------------------------- # START WITH PM2 (force Node $NODE_VERSION interpreter) # ----------------------------- cd "$LIVE_DIR" # Stop old ERP frontend process if exists $PM2_BIN delete "$APP_NAME" || true # Start ERP frontend using Node 22 interpreter $PM2_BIN start ecosystem.config.cjs \ --only "$APP_NAME" \ --update-env \ --interpreter "$NODE_BIN" # Start or restart mediaserver (ensure it's running with correct config) cd "$MEDIASERVER_DIR" if $PM2_BIN describe "$MEDIASERVER_NAME" > /dev/null 2>&1; then echo "Restarting existing mediaserver..." $PM2_BIN restart "$MEDIASERVER_NAME" --update-env --interpreter "$NODE_BIN" else echo "Starting mediaserver for the first time..." $PM2_BIN start ecosystem.config.js --update-env --interpreter "$NODE_BIN" fi # Save PM2 state (both apps) $PM2_BIN save --force # Ensure PM2 resurrects at boot WITH the right PATH/NVM $PM2_BIN startup systemd -u root --hp /root # Create a systemd drop-in so PM2 finds Node on boot AND starts only after the # iDempiere REST API (8443) answers — without the wait, every reboot produces a # ~2 minute 502 storm in the frontend until the API is up. # Keep this block in sync with deploy/pm2-root.service.d/override.conf. mkdir -p /etc/systemd/system/pm2-root.service.d cat >/etc/systemd/system/pm2-root.service.d/override.conf <