diff --git a/assets/pasted-20260203-130644-02f333fa.jpg b/assets/pasted-20260203-130644-02f333fa.jpg new file mode 100644 index 0000000..e8d19ec Binary files /dev/null and b/assets/pasted-20260203-130644-02f333fa.jpg differ diff --git a/backend/src/db/api/users.js b/backend/src/db/api/users.js index 5cb8580..8616900 100644 --- a/backend/src/db/api/users.js +++ b/backend/src/db/api/users.js @@ -226,8 +226,8 @@ module.exports = class UsersDBApi { service: item.service || null, position: item.position || null, team: item.team || null, - departmentId: item.department || item.departmentId || null, - app_roleId: item.app_role || item.app_roleId || null, + departmentId: item.departmentId || (item.department && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(item.department) ? item.department : null), + app_roleId: item.app_roleId || (item.app_role && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(item.app_role) ? item.app_role : null), importHash: item.importHash || null, createdById: currentUser.id, diff --git a/backend/src/index.js b/backend/src/index.js index 51d9d09..c5a3761 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -84,7 +84,9 @@ app.use('/api-docs', function (req, res, next) { app.use(cors({origin: true})); require('./auth/auth'); -app.use(bodyParser.json()); +// Increase JSON body limit to 50MB to match file upload limits and prevent Broken Pipe +app.use(bodyParser.json({ limit: '50mb' })); +app.use(bodyParser.urlencoded({ limit: '50mb', extended: true })); app.use('/api/auth', authRoutes); app.use('/api/file', fileRoutes); diff --git a/backend/src/routes/users.js b/backend/src/routes/users.js index a5adc80..c4a5733 100644 --- a/backend/src/routes/users.js +++ b/backend/src/routes/users.js @@ -3,7 +3,8 @@ const express = require('express'); const UsersService = require('../services/users'); const UsersDBApi = require('../db/api/users'); const wrapAsync = require('../helpers').wrapAsync; - +const db = require('../db/models'); +const processFile = require("../middlewares/upload"); const router = express.Router(); @@ -85,7 +86,7 @@ router.use(checkCrudPermissions('users')); router.post('/', wrapAsync(async (req, res) => { const referer = req.headers.referer || `${req.protocol}://${req.hostname}${req.originalUrl}`; const link = new URL(referer); - await UsersService.create(req.body.data, req.currentUser, true, link.host); + await UsersService.create(req.body.data, req.currentUser, false, link.host); const payload = true; res.status(200).send(payload); })); @@ -126,11 +127,40 @@ router.post('/', wrapAsync(async (req, res) => { * */ router.post('/bulk-import', wrapAsync(async (req, res) => { + console.log('Received bulk import request'); + + // Process file upload first + try { + await processFile(req, res); + } catch (e) { + console.error('File upload error:', e); + return res.status(400).send({ message: 'Error uploading file' }); + } + + if (!req.file || !req.file.buffer) { + return res.status(400).send({ message: 'File is required' }); + } + + // Create a job entry + const job = await db.import_jobs.create({ + filename: req.file.originalname, + status: 'pending', + imported_byId: req.currentUser.id, + imported_at: new Date(), + rows_processed: 0 + }); + const referer = req.headers.referer || `${req.protocol}://${req.hostname}${req.originalUrl}`; const link = new URL(referer); - await UsersService.bulkImport(req, res, true, link.host); - const payload = true; - res.status(200).send(payload); + + // Respond immediately to prevent timeout + res.status(200).send({ success: true, jobId: job.id }); + + // Process in background + console.log(`Starting background processing for job ${job.id}`); + UsersService.processBulkImport(req.file.buffer, req.currentUser, false, link.host, job.id) + .then(() => console.log(`Job ${job.id} finished successfully`)) + .catch(err => console.error(`Job ${job.id} failed:`, err)); })); /** diff --git a/backend/src/services/users.js b/backend/src/services/users.js index b2ad4d2..fc7eead 100644 --- a/backend/src/services/users.js +++ b/backend/src/services/users.js @@ -52,20 +52,29 @@ module.exports = class UsersService { } } - static async bulkImport(req, res, sendInvitationEmails = true, host) { - console.log('Starting bulk import...'); - + static async bulkImport(req, res, sendInvitationEmails = false, host) { + console.log('Starting bulk import (legacy called)...'); try { - await processFile(req, res); - + if (!req.file) { await processFile(req, res); } if (!req.file || !req.file.buffer) { throw new ValidationError('importer.errors.fileRequired'); } + return await this.processBulkImport(req.file.buffer, req.currentUser, sendInvitationEmails, host); + } catch (error) { + console.error('Bulk import error:', error); + throw error; + } + } - console.log('File received, size:', req.file.size); + static async processBulkImport(fileBuffer, currentUser, sendInvitationEmails = false, host, jobId = null) { + console.log(`Processing bulk import${jobId ? ` for job ${jobId}` : ''}...`); + + let totalProcessed = 0; + let totalSkipped = 0; + try { // Detect separator - const content = req.file.buffer.toString('utf-8'); + const content = fileBuffer.toString('utf-8'); const firstLine = content.split('\n')[0]; let separator = ','; if (firstLine.includes(';')) separator = ';'; @@ -88,177 +97,185 @@ module.exports = class UsersService { const defaultRoleName = config.roles?.user || 'Employee'; const userRole = await db.roles.findOne({ where: { name: defaultRoleName } }); + const normalizeHeader = (h) => { + if (!h) return ''; + return h.toLowerCase() + .normalize("NFD") + .replace(/[̀-ͯ]/g, "") // Remove diacritics + .replace(/[^a-z0-9]/g, ""); // Remove non-alphanumeric + }; + const headerMapping = { 'email': 'email', - 'e-mail': 'email', - 'mail professionnel': 'email', - 'prénom': 'firstName', + 'mailprofessionnel': 'email', + 'courriel': 'email', + 'mail': 'email', + 'adressemail': 'email', 'prenom': 'firstName', + 'firstname': 'firstName', 'nom': 'lastName', - 'téléphone': 'phoneNumber', + 'lastname': 'lastName', 'telephone': 'phoneNumber', - 'n° tel': 'phoneNumber', + 'ntel': 'phoneNumber', + 'numerotel': 'phoneNumber', + 'phonenumber': 'phoneNumber', 'matricule': 'matriculePaie', - 'matricule paie': 'matriculePaie', - 'wd id': 'workdayId', + 'matriculepaie': 'matriculePaie', + 'wdid': 'workdayId', 'workday': 'workdayId', + 'workdayid': 'workdayId', 'site': 'productionSite', - 'site de production': 'productionSite', - 'télétravail': 'remoteWork', + 'sitedeproduction': 'productionSite', + 'productionsite': 'productionSite', 'teletravail': 'remoteWork', - 'date d\'embauche': 'hiringDate', + 'remotework': 'remoteWork', + 'datedembauche': 'hiringDate', 'embauche': 'hiringDate', - 'date d\'entrée': 'positionEntryDate', - 'date d\'entrée poste': 'positionEntryDate', - 'entrée poste': 'positionEntryDate', - 'date de départ': 'departureDate', - 'départ': 'departureDate', + 'hiringdate': 'hiringDate', + 'datedentree': 'positionEntryDate', + 'datedentreeposte': 'positionEntryDate', + 'entreeposte': 'positionEntryDate', + 'positionentrydate': 'positionEntryDate', + 'datededepart': 'departureDate', + 'depart': 'departureDate', + 'departuredate': 'departureDate', 'service': 'service', 'poste': 'position', - 'équipe': 'team', + 'position': 'position', 'equipe': 'team', - 'équipe (n+1)': 'team', - 'département': 'department', - 'departement': 'department' + 'equipen1': 'team', + 'team': 'team', + 'departement': 'department', + 'department': 'department' }; const bufferStream = new stream.PassThrough(); + bufferStream.end(fileBuffer); + + const parser = bufferStream.pipe(csv({ + separator: separator, + mapHeaders: ({ header }) => { + const cleanHeader = normalizeHeader(header); + if (headerMapping[cleanHeader]) { + return headerMapping[cleanHeader]; + } + return null; + } + })); + let currentBatch = []; - const batchSize = 1000; - let totalProcessed = 0; + const batchSize = 500; let emailsToInvite = []; const processBatch = async (batch) => { if (batch.length === 0) return; - const transaction = await db.sequelize.transaction(); try { await UsersDBApi.bulkImport(batch, { transaction, ignoreDuplicates: true, - validate: false, // Disable validation for speed in large imports - currentUser: req.currentUser + validate: false, + currentUser: currentUser }); await transaction.commit(); totalProcessed += batch.length; + if (jobId) { + await db.import_jobs.update( + { rows_processed: totalProcessed }, + { where: { id: jobId } } + ); + } console.log(`Processed batch of ${batch.length}. Total processed: ${totalProcessed}`); } catch (error) { await transaction.rollback(); console.error('Batch processing error:', error); - // Continue with next batch? For now, we stop on first error in batch throw error; } }; - const parsePromise = new Promise((resolve, reject) => { - bufferStream - .pipe(csv({ - separator: separator, - mapHeaders: ({ header }) => { - const lowerHeader = header.toLowerCase().trim(); - const cleanHeader = lowerHeader.replace(/^\uFEFF/, ''); - return headerMapping[cleanHeader] || cleanHeader; + for await (const data of parser) { + Object.keys(data).forEach(key => { + if (typeof data[key] === 'string') { + data[key] = data[key].trim(); + if (data[key] === '') data[key] = null; + } + }); + + const email = data.email?.toLowerCase().trim(); + if (email && email.includes('@')) { + if (!existingEmails.has(email)) { + if (data.department) { + const deptName = data.department.toLowerCase().trim(); + if (departmentMap[deptName]) { + data.departmentId = departmentMap[deptName]; + } } - })) - .on('data', async (data) => { - // Clean up data - Object.keys(data).forEach(key => { - if (typeof data[key] === 'string') { - data[key] = data[key].trim(); - if (data[key] === '') data[key] = null; + if (data.remoteWork) { + const val = data.remoteWork.toLowerCase().trim(); + if (['oui', 'yes', 'y', 'true'].includes(val)) data.remoteWork = 'Oui'; + else if (['non', 'no', 'n', 'false'].includes(val)) data.remoteWork = 'Non'; + } + ['hiringDate', 'positionEntryDate', 'departureDate'].forEach(field => { + if (data[field]) { + const parsedDate = moment(data[field], ['DD/MM/YYYY', 'YYYY-MM-DD', 'MM/DD/YYYY'], true); + data[field] = parsedDate.isValid() ? parsedDate.toDate() : null; } }); - - const email = data.email?.toLowerCase().trim(); - - if (email && email.includes('@') && !existingEmails.has(email)) { - if (data.department) { - const deptName = data.department.toLowerCase().trim(); - if (departmentMap[deptName]) { - data.departmentId = departmentMap[deptName]; - } - } - - if (data.remoteWork) { - const val = data.remoteWork.toLowerCase().trim(); - if (['oui', 'yes', 'y', 'true'].includes(val)) data.remoteWork = 'Oui'; - else if (['non', 'no', 'n', 'false'].includes(val)) data.remoteWork = 'Non'; - } - - ['hiringDate', 'positionEntryDate', 'departureDate'].forEach(field => { - if (data[field]) { - const parsedDate = moment(data[field], ['DD/MM/YYYY', 'YYYY-MM-DD', 'MM/DD/YYYY'], true); - data[field] = parsedDate.isValid() ? parsedDate.toDate() : null; - } - }); - - if (!data.app_roleId && userRole) { - data.app_roleId = userRole.id; - } - - currentBatch.push(data); - existingEmails.add(email); - emailsToInvite.push(email); - - if (currentBatch.length >= batchSize) { - const batchToProcess = [...currentBatch]; - currentBatch = []; - // We need to pause the stream to wait for the batch to be processed - bufferStream.pause(); - processBatch(batchToProcess) - .then(() => bufferStream.resume()) - .catch(err => { - bufferStream.destroy(err); - reject(err); - }); - } + if (!data.app_roleId && userRole) { + data.app_roleId = userRole.id; } - }) - .on('end', async () => { - try { - if (currentBatch.length > 0) { - await processBatch(currentBatch); - } - console.log('CSV parsing and batch processing finished. Total:', totalProcessed); - resolve(); - } catch (err) { - reject(err); + + currentBatch.push(data); + existingEmails.add(email); + emailsToInvite.push(email); + + if (currentBatch.length >= batchSize) { + await processBatch(currentBatch); + currentBatch = []; } - }) - .on('error', (error) => { - console.error('CSV parsing error:', error); - reject(error); - }); - }); - - bufferStream.end(req.file.buffer); - await parsePromise; - - if (totalProcessed === 0) { - throw new ValidationError('importer.errors.noRowsFound'); - } - - // Send emails in background to avoid blocking the response further - if (emailsToInvite.length > 0 && sendInvitationEmails) { - console.log(`Starting background email sending for ${emailsToInvite.length} users...`); - // Use a simple background loop with delays to avoid overwhelming SMTP - const sendEmailsInBackground = async (emails) => { - const batchSize = 50; - for (let i = 0; i < emails.length; i += batchSize) { - const batch = emails.slice(i, i + batchSize); - await Promise.all(batch.map(email => - AuthService.sendPasswordResetEmail(email, 'invitation', host).catch(err => console.error(`Failed to send email to ${email}:`, err)) - )); - console.log(`Sent email batch ${i / batchSize + 1}. Total sent: ${Math.min(i + batchSize, emails.length)}`); - // Small delay between batches - await new Promise(resolve => setTimeout(resolve, 1000)); + } else { + totalSkipped++; } - }; - sendEmailsInBackground(emailsToInvite); + } } + if (currentBatch.length > 0) { + await processBatch(currentBatch); + } + + if (jobId) { + await db.import_jobs.update( + { status: 'completed', rows_processed: totalProcessed }, + { where: { id: jobId } } + ); + } + + console.log('Import finished. Total:', totalProcessed, 'Skipped:', totalSkipped); + + // Send emails in background + if (emailsToInvite.length > 0 && sendInvitationEmails) { + (async () => { + const batchSize = 50; + for (let i = 0; i < emailsToInvite.length; i += batchSize) { + const batch = emailsToInvite.slice(i, i + batchSize); + await Promise.all(batch.map(email => + AuthService.sendPasswordResetEmail(email, 'invitation', host).catch(err => console.error(`Email error for ${email}:`, err)) + )); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + })().catch(err => console.error('Background email error:', err)); + } + + return { totalProcessed, totalSkipped }; + } catch (error) { console.error('Bulk import error:', error); + if (jobId) { + await db.import_jobs.update( + { status: 'failed' }, + { where: { id: jobId } } + ).catch(e => console.error('Failed to update job status:', e)); + } throw error; } } diff --git a/frontend/src/menuAside.ts b/frontend/src/menuAside.ts index 1366712..1bdc8a5 100644 --- a/frontend/src/menuAside.ts +++ b/frontend/src/menuAside.ts @@ -5,82 +5,87 @@ const menuAside: MenuAsideItem[] = [ { href: '/dashboard', icon: icon.mdiViewDashboardOutline, - label: 'Dashboard', + label: 'Tableau de bord', }, { href: '/check-in', - label: 'Pointage Entrée', + label: 'Enregistrement Entrée', icon: icon.mdiClockIn, permissions: 'CREATE_TIME_ENTRIES' }, { - href: '/users/users-list', - label: 'Collaborateurs', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: icon.mdiAccountGroup ?? icon.mdiTable, - permissions: 'READ_USERS' + label: 'Données', + icon: icon.mdiDatabase, + menu: [ + { + href: '/users/users-list', + label: 'Collaborateurs', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: icon.mdiAccountGroup ?? icon.mdiTable, + permissions: 'READ_USERS' + }, + { + href: '/time_entries/time_entries-list', + label: 'Historique Entrées', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: 'mdiClock' in icon ? icon['mdiClock' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, + permissions: 'READ_TIME_ENTRIES' + }, + { + href: '/badges/badges-list', + label: 'Badges', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: 'mdiBadgeAccount' in icon ? icon['mdiBadgeAccount' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, + permissions: 'READ_BADGES' + }, + ] }, { - href: '/time_entries/time_entries-list', - label: 'Présences', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: 'mdiClock' in icon ? icon['mdiClock' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, - permissions: 'READ_TIME_ENTRIES' - }, - { - href: '/roles/roles-list', - label: 'Roles', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: icon.mdiShieldAccountVariantOutline ?? icon.mdiTable, - permissions: 'READ_ROLES' - }, - { - href: '/permissions/permissions-list', - label: 'Permissions', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: icon.mdiShieldAccountOutline ?? icon.mdiTable, - permissions: 'READ_PERMISSIONS' - }, - { - href: '/departments/departments-list', - label: 'Departments', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: 'mdiOfficeBuilding' in icon ? icon['mdiOfficeBuilding' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, - permissions: 'READ_DEPARTMENTS' - }, - { - href: '/import_jobs/import_jobs-list', - label: 'Import jobs', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: 'mdiFileImport' in icon ? icon['mdiFileImport' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, - permissions: 'READ_IMPORT_JOBS' - }, - { - href: '/badges/badges-list', - label: 'Badges', - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - icon: 'mdiBadgeAccount' in icon ? icon['mdiBadgeAccount' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, - permissions: 'READ_BADGES' + label: 'Configuration', + icon: icon.mdiCog, + menu: [ + { + href: '/import_jobs/import_jobs-list', + label: 'Imports CSV', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: 'mdiFileImport' in icon ? icon['mdiFileImport' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, + permissions: 'READ_IMPORT_JOBS' + }, + { + href: '/departments/departments-list', + label: 'Départements', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: 'mdiOfficeBuilding' in icon ? icon['mdiOfficeBuilding' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable, + permissions: 'READ_DEPARTMENTS' + }, + { + href: '/roles/roles-list', + label: 'Rôles', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: icon.mdiShieldAccountVariantOutline ?? icon.mdiTable, + permissions: 'READ_ROLES' + }, + { + href: '/permissions/permissions-list', + label: 'Permissions', + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + icon: icon.mdiShieldAccountOutline ?? icon.mdiTable, + permissions: 'READ_PERMISSIONS' + }, + ] }, { href: '/profile', - label: 'Profile', + label: 'Mon Profil', icon: icon.mdiAccountCircle, }, - { - href: '/api-docs', - target: '_blank', - label: 'Swagger API', - icon: icon.mdiFileCode, - permissions: 'READ_API_DOCS' - }, ] -export default menuAside \ No newline at end of file +export default menuAside diff --git a/frontend/src/pages/check-in.tsx b/frontend/src/pages/check-in.tsx index 788527c..6be3cf6 100644 --- a/frontend/src/pages/check-in.tsx +++ b/frontend/src/pages/check-in.tsx @@ -2,7 +2,7 @@ import { mdiClockIn, mdiAccountSearch, mdiCalendarClock, mdiCardAccountDetailsOu import Head from 'next/head' import React, { ReactElement, useEffect, useState } from 'react' import CardBox from '../components/CardBox' -import LayoutGuest from '../layouts/Guest' +import LayoutAuthenticated from '../layouts/Authenticated' import SectionMain from '../components/SectionMain' import SectionTitleLineWithButton from '../components/SectionTitleLineWithButton' import { getPageTitle } from '../config' @@ -64,7 +64,7 @@ const EmployeeDetails = () => {

Nom & Prénom:

-

{employeeData.firstName} {employeeData.lastName}

+

{employeeData.firstName} {employeeData.lastName}

Département:

@@ -75,7 +75,7 @@ const EmployeeDetails = () => {

{employeeData.matriculePaie || '-'}

-

Workday ID:

+

WD ID:

{employeeData.workdayId || '-'}

@@ -106,7 +106,7 @@ const EmployeeDetails = () => { const CheckIn = () => { const dispatch = useAppDispatch() const [isSuccess, setIsSuccess] = useState(false) - const [error, setError] = useState(null) + const [error, setError] = useState(null) const handleSubmit = async (data, { resetForm }) => { setIsSuccess(false) @@ -116,6 +116,8 @@ const CheckIn = () => { if (create.fulfilled.match(resultAction)) { setIsSuccess(true) resetForm() + // Keep the start_time current + // data.start_time = moment().format('YYYY-MM-DDTHH:mm') } else { setError("Une erreur est survenue lors de l'enregistrement.") } @@ -153,14 +155,14 @@ const CheckIn = () => { > {({ isSubmitting }) => (
- + @@ -168,13 +170,15 @@ const CheckIn = () => { - - - +
+ + + - - - + + + +
@@ -182,7 +186,7 @@ const CheckIn = () => { { CheckIn.getLayout = function getLayout(page: ReactElement) { return ( - + {page} - + ) } -export default CheckIn +export default CheckIn \ No newline at end of file