311 lines
11 KiB
TypeScript
311 lines
11 KiB
TypeScript
import { Router } from 'express';
|
|
import { eq, getTableColumns } from 'drizzle-orm';
|
|
import multer from 'multer';
|
|
import { db } from '../db/client';
|
|
import { rabbits, litters } from '../db/schema';
|
|
import { toCamelCase, pick } from '../utils/case';
|
|
|
|
const router = Router();
|
|
|
|
// In-memory upload: import files are small (a herd of a few thousand rabbits is
|
|
// well under a megabyte) and we parse the whole thing in one shot, so there's no
|
|
// reason to spill to disk. 10 MB is a generous ceiling that still rejects a
|
|
// pathological upload before we try to buffer it.
|
|
const upload = multer({
|
|
storage: multer.memoryStorage(),
|
|
limits: { fileSize: 10 * 1024 * 1024 },
|
|
}).single('file');
|
|
|
|
const VALID_SEX = new Set(['buck', 'doe', 'unknown']);
|
|
const VALID_STATUS = new Set(['active', 'for_sale', 'on_hold', 'sold', 'deceased', 'other']);
|
|
|
|
// Legacy Kindlebox (PHP app) numeric status codes → MyKindlebox status values.
|
|
// Sourced from the old editrabbit.php status <select> (0 Herd / 1 For Sale /
|
|
// 2 Pending Sale / 3 Sold / 4 Other / 5 Dead). Any unmapped/garbage code falls
|
|
// back to 'active' so a stray value never blocks an import.
|
|
const LEGACY_STATUS_MAP: Record<string, string> = {
|
|
'0': 'active',
|
|
'1': 'for_sale',
|
|
'2': 'on_hold',
|
|
'3': 'sold',
|
|
'4': 'other',
|
|
'5': 'deceased',
|
|
};
|
|
|
|
// Only these columns can be set from an imported row. `id` is allowed (so a
|
|
// native re-import / legacy import can preserve original ids and therefore
|
|
// parentage); timestamps and photo_path stay server-managed. sireId/damId are
|
|
// handled separately via the two-pass FK resolution below, so they're excluded
|
|
// here and re-attached after the id set is known.
|
|
const RABBIT_SYSTEM_FIELDS = new Set(['createdAt', 'updatedAt', 'photoPath', 'sireId', 'damId']);
|
|
const RABBIT_IMPORTABLE = Object.keys(getTableColumns(rabbits)).filter(
|
|
(key) => !RABBIT_SYSTEM_FIELDS.has(key)
|
|
);
|
|
|
|
interface NormalizedRabbit {
|
|
values: Record<string, unknown>; // camelCase, ready for drizzle insert (no sire/dam FK)
|
|
id: number | null;
|
|
sireRaw: number | null;
|
|
damRaw: number | null;
|
|
}
|
|
|
|
function toInt(value: unknown): number | null {
|
|
if (value === null || value === undefined || value === '') return null;
|
|
const n = Number.parseInt(String(value), 10);
|
|
return Number.isNaN(n) ? null : n;
|
|
}
|
|
|
|
// Legacy "0" sentinels (and blanks) mean "not recorded" for ear/registration
|
|
// numbers — store them as null rather than a literal "0".
|
|
function cleanSentinel(value: unknown): string | null {
|
|
if (value === null || value === undefined) return null;
|
|
const s = String(value).trim();
|
|
return s === '' || s === '0' ? null : s;
|
|
}
|
|
|
|
function toWeight(value: unknown): number | null {
|
|
if (value === null || value === undefined || value === '') return null;
|
|
const n = Number.parseFloat(String(value));
|
|
// 0 means "unweighed" in the legacy data; treat it as missing.
|
|
return Number.isNaN(n) || n === 0 ? null : n;
|
|
}
|
|
|
|
function cleanGene(value: unknown): string | null {
|
|
if (value === null || value === undefined) return null;
|
|
const s = String(value).trim();
|
|
return s === '' ? null : s;
|
|
}
|
|
|
|
// --- CSV parsing (RFC-4180-ish: quoted fields, "" escapes, CR/LF) ---
|
|
|
|
function parseCsv(text: string): string[][] {
|
|
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); // strip BOM
|
|
const rows: string[][] = [];
|
|
let row: string[] = [];
|
|
let field = '';
|
|
let inQuotes = false;
|
|
let i = 0;
|
|
|
|
while (i < text.length) {
|
|
const c = text[i];
|
|
if (inQuotes) {
|
|
if (c === '"') {
|
|
if (text[i + 1] === '"') {
|
|
field += '"';
|
|
i += 2;
|
|
continue;
|
|
}
|
|
inQuotes = false;
|
|
i++;
|
|
continue;
|
|
}
|
|
field += c;
|
|
i++;
|
|
continue;
|
|
}
|
|
if (c === '"') {
|
|
inQuotes = true;
|
|
i++;
|
|
} else if (c === ',') {
|
|
row.push(field);
|
|
field = '';
|
|
i++;
|
|
} else if (c === '\r') {
|
|
i++;
|
|
} else if (c === '\n') {
|
|
row.push(field);
|
|
rows.push(row);
|
|
row = [];
|
|
field = '';
|
|
i++;
|
|
} else {
|
|
field += c;
|
|
i++;
|
|
}
|
|
}
|
|
if (field.length > 0 || row.length > 0) {
|
|
row.push(field);
|
|
rows.push(row);
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
function csvToObjects(text: string): Record<string, string>[] {
|
|
const rows = parseCsv(text).filter((r) => !(r.length === 1 && r[0].trim() === ''));
|
|
if (rows.length === 0) return [];
|
|
const header = rows[0].map((h) => h.trim());
|
|
return rows.slice(1).map((cells) => {
|
|
const obj: Record<string, string> = {};
|
|
header.forEach((h, idx) => {
|
|
obj[h] = cells[idx] ?? '';
|
|
});
|
|
return obj;
|
|
});
|
|
}
|
|
|
|
// --- Row normalization ---
|
|
|
|
function isLegacyHeader(keys: string[]): boolean {
|
|
const set = new Set(keys);
|
|
return set.has('ID') && set.has('GENDER') && set.has('SIRE_ID');
|
|
}
|
|
|
|
function normalizeLegacy(o: Record<string, string>): NormalizedRabbit {
|
|
const gender = String(o.GENDER ?? '').trim();
|
|
return {
|
|
id: toInt(o.ID),
|
|
sireRaw: toInt(o.SIRE_ID),
|
|
damRaw: toInt(o.DAM_ID),
|
|
values: {
|
|
id: toInt(o.ID),
|
|
name: (o.NAME ?? '').trim() || 'Unnamed',
|
|
sex: gender === '1' ? 'buck' : gender === '0' ? 'doe' : 'unknown',
|
|
dob: (o.DATE_OF_BIRTH ?? '').trim() || null,
|
|
earNumber: cleanSentinel(o.EAR_NUMBER),
|
|
colorName: (o.COLOR ?? '').trim() || null,
|
|
status: LEGACY_STATUS_MAP[String(o.STATUS ?? '').trim()] ?? 'active',
|
|
weightLbs: toWeight(o.WEIGHT),
|
|
registrationNumber: cleanSentinel(o.REG_NUM),
|
|
gcNumber: cleanSentinel(o.GC_NUM),
|
|
farm: (o.FARM_NAME ?? '').trim() || null,
|
|
locusA: cleanGene(o.A_GENE),
|
|
locusB: cleanGene(o.B_GENE),
|
|
locusC: cleanGene(o.C_GENE),
|
|
locusD: cleanGene(o.D_GENE),
|
|
locusE: cleanGene(o.E_GENE),
|
|
locusEn: cleanGene(o.EN_GENE),
|
|
locusV: cleanGene(o.V_GENE),
|
|
},
|
|
};
|
|
}
|
|
|
|
// Native format: a row whose keys already match the API's snake_case (what this
|
|
// app exports), or an equivalent JSON object. Unknown keys are dropped; sex /
|
|
// status are clamped to valid enum values so a malformed row can't 500 against
|
|
// the DB CHECK constraint.
|
|
function normalizeNative(o: Record<string, unknown>): NormalizedRabbit {
|
|
const camel = pick(toCamelCase(o as Record<string, unknown>), RABBIT_IMPORTABLE);
|
|
|
|
const id = toInt((o as Record<string, unknown>).id);
|
|
const sireRaw = toInt((o as Record<string, unknown>).sire_id ?? (o as Record<string, unknown>).sireId);
|
|
const damRaw = toInt((o as Record<string, unknown>).dam_id ?? (o as Record<string, unknown>).damId);
|
|
|
|
const values: Record<string, unknown> = { ...camel };
|
|
values.id = id;
|
|
values.name = (values.name != null && String(values.name).trim() !== '') ? values.name : 'Unnamed';
|
|
values.sex = VALID_SEX.has(String(values.sex)) ? values.sex : 'unknown';
|
|
values.status = VALID_STATUS.has(String(values.status)) ? values.status : 'active';
|
|
if (values.weightLbs !== undefined) values.weightLbs = toWeight(values.weightLbs);
|
|
|
|
return { id, sireRaw, damRaw, values };
|
|
}
|
|
|
|
router.post('/', (req, res) => {
|
|
upload(req, res, (err) => {
|
|
if (err) {
|
|
res.status(400).json({ error: 'Upload failed (file too large or malformed)' });
|
|
return;
|
|
}
|
|
|
|
const file = req.file;
|
|
if (!file) {
|
|
res.status(400).json({ error: 'No file provided' });
|
|
return;
|
|
}
|
|
|
|
const text = file.buffer.toString('utf8');
|
|
|
|
// A NUL byte means the file isn't UTF-8 text — almost always a UTF-16 export
|
|
// (Excel "Unicode Text") or an actual binary file (.xlsx/.xls/image). Decoding
|
|
// it as UTF-8 produces garbage that would otherwise import as phantom rows, so
|
|
// reject it with an actionable message instead.
|
|
if (file.buffer.includes(0)) {
|
|
res.status(400).json({
|
|
error:
|
|
"This doesn't look like a UTF-8 text file. If you exported from Excel, re-save as \"CSV UTF-8\" (not \"Unicode Text\" or .xlsx).",
|
|
});
|
|
return;
|
|
}
|
|
|
|
const isJson =
|
|
file.originalname.toLowerCase().endsWith('.json') ||
|
|
text.trimStart().startsWith('[') ||
|
|
text.trimStart().startsWith('{');
|
|
|
|
let normalized: NormalizedRabbit[];
|
|
let format: string;
|
|
|
|
try {
|
|
if (isJson) {
|
|
const parsed = JSON.parse(text);
|
|
const arr: unknown[] = Array.isArray(parsed)
|
|
? parsed
|
|
: Array.isArray((parsed as Record<string, unknown>)?.rabbits)
|
|
? ((parsed as Record<string, unknown>).rabbits as unknown[])
|
|
: [];
|
|
if (!Array.isArray(arr) || arr.length === 0) {
|
|
res.status(400).json({ error: 'JSON must be an array of rabbits, or { "rabbits": [...] }' });
|
|
return;
|
|
}
|
|
normalized = arr.map((row) => normalizeNative(row as Record<string, unknown>));
|
|
format = 'json';
|
|
} else {
|
|
const objects = csvToObjects(text);
|
|
if (objects.length === 0) {
|
|
res.status(400).json({ error: 'CSV has no data rows' });
|
|
return;
|
|
}
|
|
const legacy = isLegacyHeader(Object.keys(objects[0]));
|
|
normalized = objects.map((o) => (legacy ? normalizeLegacy(o) : normalizeNative(o)));
|
|
format = legacy ? 'legacy-csv' : 'csv';
|
|
}
|
|
} catch {
|
|
res.status(400).json({ error: 'Could not parse file — expected valid CSV or JSON' });
|
|
return;
|
|
}
|
|
|
|
// Resolve parentage: keep a sire/dam FK only when it points at an id that's
|
|
// actually present in this import; otherwise drop it (legacy 9999 sentinels,
|
|
// or references to rabbits that weren't included).
|
|
const idSet = new Set<number>();
|
|
for (const r of normalized) {
|
|
if (r.id != null) idSet.add(r.id);
|
|
}
|
|
|
|
try {
|
|
db.transaction((tx) => {
|
|
// Litters reference rabbits, so they must go first — and they're wiped
|
|
// regardless of the "rabbits only" import scope, since leaving them
|
|
// would dangle against deleted rabbits.
|
|
tx.delete(litters).run();
|
|
tx.delete(rabbits).run();
|
|
|
|
// Pass 1: insert every rabbit with no parent links yet (a child may
|
|
// appear before its parent in the file).
|
|
for (const r of normalized) {
|
|
tx.insert(rabbits)
|
|
.values({ ...r.values, sireId: null, damId: null } as typeof rabbits.$inferInsert)
|
|
.run();
|
|
}
|
|
|
|
// Pass 2: attach FKs now that all ids exist.
|
|
for (const r of normalized) {
|
|
if (r.id == null) continue;
|
|
const sireId = r.sireRaw != null && idSet.has(r.sireRaw) ? r.sireRaw : null;
|
|
const damId = r.damRaw != null && idSet.has(r.damRaw) ? r.damRaw : null;
|
|
if (sireId === null && damId === null) continue;
|
|
tx.update(rabbits).set({ sireId, damId }).where(eq(rabbits.id, r.id)).run();
|
|
}
|
|
});
|
|
} catch (e) {
|
|
console.error('Import failed:', e);
|
|
res.status(500).json({ error: 'Import failed — no changes were made' });
|
|
return;
|
|
}
|
|
|
|
res.json({ format, rabbits_imported: normalized.length });
|
|
});
|
|
});
|
|
|
|
export default router;
|