From 4b9b8b726cbb38b251edc05d4789d0aab44cb87d Mon Sep 17 00:00:00 2001 From: ChrisChrome Date: Sun, 22 Dec 2024 17:16:53 -0700 Subject: [PATCH] Add dedupe --- routes/admin.js | 8 +++++++- views/admin/import.ejs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/routes/admin.js b/routes/admin.js index 48e9e1a..a62bd3b 100644 --- a/routes/admin.js +++ b/routes/admin.js @@ -169,7 +169,7 @@ router.post('/import', authenticate, upload.single('fileInput'), async (req, res const data = fs.readFileSync(filePath, 'utf-8'); const lines = data.split('\n'); const conn = await pool.getConnection(); - uploads[uploadId] = { completed: false, total: lines.length, processed: 0 }; + uploads[uploadId] = { completed: false, total: lines.length, processed: 0, skipped: 0 }; res.end(JSON.stringify({ success: true, message: 'Started upload, please wait!', id: uploadId })); for (const line of lines) { // This is a text file, split by "x/"", get [0], then split by / and get the last element, that will be an ID. All other values are static. @@ -178,6 +178,12 @@ router.post('/import', authenticate, upload.single('fileInput'), async (req, res const reasonShort = "Listed by MFD"; const moderator = "MFD"; const reasonsFlag = flags.addFlag(0, reasonFlags.CHILD_SAFETY) + const existingBan = await conn.query('SELECT * FROM bans WHERE robloxId = ?', [robloxId]); + if (existingBan.length > 0) { + uploads[uploadId].processed++; + uploads[uploadId].skipped++ + continue; // Skip this entry if robloxId already exists + } await conn.query('INSERT INTO bans (robloxId, reasonShort, reasonLong, reasonsFlag, moderator) VALUES (?, ?, ?, ?, ?)', [robloxId, reasonShort, reasonLong, reasonsFlag, moderator]); uploads[uploadId].processed++; diff --git a/views/admin/import.ejs b/views/admin/import.ejs index 3cef723..e032b70 100644 --- a/views/admin/import.ejs +++ b/views/admin/import.ejs @@ -15,7 +15,7 @@

Mass Ban (Import)

- The import system does NOT have deduplication. Please be sure to remove duplicate entries from your list BEFORE uploading! + The import system does NOT have deduplication for CSV files. Please be sure to remove duplicate entries from your list BEFORE uploading!