Auto commit: 2026-02-03T14:34:33.216Z

This commit is contained in:
Flatlogic Bot 2026-02-03 14:34:33 +00:00
parent 6ee47994e6
commit fadb1454c6
7 changed files with 277 additions and 219 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 50 KiB

View File

@ -226,8 +226,8 @@ module.exports = class UsersDBApi {
service: item.service || null,
position: item.position || null,
team: item.team || null,
departmentId: item.department || item.departmentId || null,
app_roleId: item.app_role || item.app_roleId || null,
departmentId: item.departmentId || (item.department && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(item.department) ? item.department : null),
app_roleId: item.app_roleId || (item.app_role && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(item.app_role) ? item.app_role : null),
importHash: item.importHash || null,
createdById: currentUser.id,

View File

@ -84,7 +84,9 @@ app.use('/api-docs', function (req, res, next) {
app.use(cors({origin: true}));
require('./auth/auth');
app.use(bodyParser.json());
// Increase JSON body limit to 50MB to match file upload limits and prevent Broken Pipe
app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
app.use('/api/auth', authRoutes);
app.use('/api/file', fileRoutes);

View File

@ -3,7 +3,8 @@ const express = require('express');
const UsersService = require('../services/users');
const UsersDBApi = require('../db/api/users');
const wrapAsync = require('../helpers').wrapAsync;
const db = require('../db/models');
const processFile = require("../middlewares/upload");
const router = express.Router();
@ -85,7 +86,7 @@ router.use(checkCrudPermissions('users'));
router.post('/', wrapAsync(async (req, res) => {
const referer = req.headers.referer || `${req.protocol}://${req.hostname}${req.originalUrl}`;
const link = new URL(referer);
await UsersService.create(req.body.data, req.currentUser, true, link.host);
await UsersService.create(req.body.data, req.currentUser, false, link.host);
const payload = true;
res.status(200).send(payload);
}));
@ -126,11 +127,40 @@ router.post('/', wrapAsync(async (req, res) => {
*
*/
router.post('/bulk-import', wrapAsync(async (req, res) => {
console.log('Received bulk import request');
// Process file upload first
try {
await processFile(req, res);
} catch (e) {
console.error('File upload error:', e);
return res.status(400).send({ message: 'Error uploading file' });
}
if (!req.file || !req.file.buffer) {
return res.status(400).send({ message: 'File is required' });
}
// Create a job entry
const job = await db.import_jobs.create({
filename: req.file.originalname,
status: 'pending',
imported_byId: req.currentUser.id,
imported_at: new Date(),
rows_processed: 0
});
const referer = req.headers.referer || `${req.protocol}://${req.hostname}${req.originalUrl}`;
const link = new URL(referer);
await UsersService.bulkImport(req, res, true, link.host);
const payload = true;
res.status(200).send(payload);
// Respond immediately to prevent timeout
res.status(200).send({ success: true, jobId: job.id });
// Process in background
console.log(`Starting background processing for job ${job.id}`);
UsersService.processBulkImport(req.file.buffer, req.currentUser, false, link.host, job.id)
.then(() => console.log(`Job ${job.id} finished successfully`))
.catch(err => console.error(`Job ${job.id} failed:`, err));
}));
/**

View File

@ -52,20 +52,29 @@ module.exports = class UsersService {
}
}
static async bulkImport(req, res, sendInvitationEmails = true, host) {
console.log('Starting bulk import...');
static async bulkImport(req, res, sendInvitationEmails = false, host) {
console.log('Starting bulk import (legacy called)...');
try {
await processFile(req, res);
if (!req.file) { await processFile(req, res); }
if (!req.file || !req.file.buffer) {
throw new ValidationError('importer.errors.fileRequired');
}
return await this.processBulkImport(req.file.buffer, req.currentUser, sendInvitationEmails, host);
} catch (error) {
console.error('Bulk import error:', error);
throw error;
}
}
console.log('File received, size:', req.file.size);
static async processBulkImport(fileBuffer, currentUser, sendInvitationEmails = false, host, jobId = null) {
console.log(`Processing bulk import${jobId ? ` for job ${jobId}` : ''}...`);
let totalProcessed = 0;
let totalSkipped = 0;
try {
// Detect separator
const content = req.file.buffer.toString('utf-8');
const content = fileBuffer.toString('utf-8');
const firstLine = content.split('\n')[0];
let separator = ',';
if (firstLine.includes(';')) separator = ';';
@ -88,177 +97,185 @@ module.exports = class UsersService {
const defaultRoleName = config.roles?.user || 'Employee';
const userRole = await db.roles.findOne({ where: { name: defaultRoleName } });
const normalizeHeader = (h) => {
if (!h) return '';
return h.toLowerCase()
.normalize("NFD")
.replace(/[̀-ͯ]/g, "") // Remove diacritics
.replace(/[^a-z0-9]/g, ""); // Remove non-alphanumeric
};
const headerMapping = {
'email': 'email',
'e-mail': 'email',
'mail professionnel': 'email',
'prénom': 'firstName',
'mailprofessionnel': 'email',
'courriel': 'email',
'mail': 'email',
'adressemail': 'email',
'prenom': 'firstName',
'firstname': 'firstName',
'nom': 'lastName',
'téléphone': 'phoneNumber',
'lastname': 'lastName',
'telephone': 'phoneNumber',
'n° tel': 'phoneNumber',
'ntel': 'phoneNumber',
'numerotel': 'phoneNumber',
'phonenumber': 'phoneNumber',
'matricule': 'matriculePaie',
'matricule paie': 'matriculePaie',
'wd id': 'workdayId',
'matriculepaie': 'matriculePaie',
'wdid': 'workdayId',
'workday': 'workdayId',
'workdayid': 'workdayId',
'site': 'productionSite',
'site de production': 'productionSite',
'télétravail': 'remoteWork',
'sitedeproduction': 'productionSite',
'productionsite': 'productionSite',
'teletravail': 'remoteWork',
'date d\'embauche': 'hiringDate',
'remotework': 'remoteWork',
'datedembauche': 'hiringDate',
'embauche': 'hiringDate',
'date d\'entrée': 'positionEntryDate',
'date d\'entrée poste': 'positionEntryDate',
'entrée poste': 'positionEntryDate',
'date de départ': 'departureDate',
'départ': 'departureDate',
'hiringdate': 'hiringDate',
'datedentree': 'positionEntryDate',
'datedentreeposte': 'positionEntryDate',
'entreeposte': 'positionEntryDate',
'positionentrydate': 'positionEntryDate',
'datededepart': 'departureDate',
'depart': 'departureDate',
'departuredate': 'departureDate',
'service': 'service',
'poste': 'position',
'équipe': 'team',
'position': 'position',
'equipe': 'team',
'équipe (n+1)': 'team',
'département': 'department',
'departement': 'department'
'equipen1': 'team',
'team': 'team',
'departement': 'department',
'department': 'department'
};
const bufferStream = new stream.PassThrough();
bufferStream.end(fileBuffer);
const parser = bufferStream.pipe(csv({
separator: separator,
mapHeaders: ({ header }) => {
const cleanHeader = normalizeHeader(header);
if (headerMapping[cleanHeader]) {
return headerMapping[cleanHeader];
}
return null;
}
}));
let currentBatch = [];
const batchSize = 1000;
let totalProcessed = 0;
const batchSize = 500;
let emailsToInvite = [];
const processBatch = async (batch) => {
if (batch.length === 0) return;
const transaction = await db.sequelize.transaction();
try {
await UsersDBApi.bulkImport(batch, {
transaction,
ignoreDuplicates: true,
validate: false, // Disable validation for speed in large imports
currentUser: req.currentUser
validate: false,
currentUser: currentUser
});
await transaction.commit();
totalProcessed += batch.length;
if (jobId) {
await db.import_jobs.update(
{ rows_processed: totalProcessed },
{ where: { id: jobId } }
);
}
console.log(`Processed batch of ${batch.length}. Total processed: ${totalProcessed}`);
} catch (error) {
await transaction.rollback();
console.error('Batch processing error:', error);
// Continue with next batch? For now, we stop on first error in batch
throw error;
}
};
const parsePromise = new Promise((resolve, reject) => {
bufferStream
.pipe(csv({
separator: separator,
mapHeaders: ({ header }) => {
const lowerHeader = header.toLowerCase().trim();
const cleanHeader = lowerHeader.replace(/^\uFEFF/, '');
return headerMapping[cleanHeader] || cleanHeader;
for await (const data of parser) {
Object.keys(data).forEach(key => {
if (typeof data[key] === 'string') {
data[key] = data[key].trim();
if (data[key] === '') data[key] = null;
}
});
const email = data.email?.toLowerCase().trim();
if (email && email.includes('@')) {
if (!existingEmails.has(email)) {
if (data.department) {
const deptName = data.department.toLowerCase().trim();
if (departmentMap[deptName]) {
data.departmentId = departmentMap[deptName];
}
}
}))
.on('data', async (data) => {
// Clean up data
Object.keys(data).forEach(key => {
if (typeof data[key] === 'string') {
data[key] = data[key].trim();
if (data[key] === '') data[key] = null;
if (data.remoteWork) {
const val = data.remoteWork.toLowerCase().trim();
if (['oui', 'yes', 'y', 'true'].includes(val)) data.remoteWork = 'Oui';
else if (['non', 'no', 'n', 'false'].includes(val)) data.remoteWork = 'Non';
}
['hiringDate', 'positionEntryDate', 'departureDate'].forEach(field => {
if (data[field]) {
const parsedDate = moment(data[field], ['DD/MM/YYYY', 'YYYY-MM-DD', 'MM/DD/YYYY'], true);
data[field] = parsedDate.isValid() ? parsedDate.toDate() : null;
}
});
const email = data.email?.toLowerCase().trim();
if (email && email.includes('@') && !existingEmails.has(email)) {
if (data.department) {
const deptName = data.department.toLowerCase().trim();
if (departmentMap[deptName]) {
data.departmentId = departmentMap[deptName];
}
}
if (data.remoteWork) {
const val = data.remoteWork.toLowerCase().trim();
if (['oui', 'yes', 'y', 'true'].includes(val)) data.remoteWork = 'Oui';
else if (['non', 'no', 'n', 'false'].includes(val)) data.remoteWork = 'Non';
}
['hiringDate', 'positionEntryDate', 'departureDate'].forEach(field => {
if (data[field]) {
const parsedDate = moment(data[field], ['DD/MM/YYYY', 'YYYY-MM-DD', 'MM/DD/YYYY'], true);
data[field] = parsedDate.isValid() ? parsedDate.toDate() : null;
}
});
if (!data.app_roleId && userRole) {
data.app_roleId = userRole.id;
}
currentBatch.push(data);
existingEmails.add(email);
emailsToInvite.push(email);
if (currentBatch.length >= batchSize) {
const batchToProcess = [...currentBatch];
currentBatch = [];
// We need to pause the stream to wait for the batch to be processed
bufferStream.pause();
processBatch(batchToProcess)
.then(() => bufferStream.resume())
.catch(err => {
bufferStream.destroy(err);
reject(err);
});
}
if (!data.app_roleId && userRole) {
data.app_roleId = userRole.id;
}
})
.on('end', async () => {
try {
if (currentBatch.length > 0) {
await processBatch(currentBatch);
}
console.log('CSV parsing and batch processing finished. Total:', totalProcessed);
resolve();
} catch (err) {
reject(err);
currentBatch.push(data);
existingEmails.add(email);
emailsToInvite.push(email);
if (currentBatch.length >= batchSize) {
await processBatch(currentBatch);
currentBatch = [];
}
})
.on('error', (error) => {
console.error('CSV parsing error:', error);
reject(error);
});
});
bufferStream.end(req.file.buffer);
await parsePromise;
if (totalProcessed === 0) {
throw new ValidationError('importer.errors.noRowsFound');
}
// Send emails in background to avoid blocking the response further
if (emailsToInvite.length > 0 && sendInvitationEmails) {
console.log(`Starting background email sending for ${emailsToInvite.length} users...`);
// Use a simple background loop with delays to avoid overwhelming SMTP
const sendEmailsInBackground = async (emails) => {
const batchSize = 50;
for (let i = 0; i < emails.length; i += batchSize) {
const batch = emails.slice(i, i + batchSize);
await Promise.all(batch.map(email =>
AuthService.sendPasswordResetEmail(email, 'invitation', host).catch(err => console.error(`Failed to send email to ${email}:`, err))
));
console.log(`Sent email batch ${i / batchSize + 1}. Total sent: ${Math.min(i + batchSize, emails.length)}`);
// Small delay between batches
await new Promise(resolve => setTimeout(resolve, 1000));
} else {
totalSkipped++;
}
};
sendEmailsInBackground(emailsToInvite);
}
}
if (currentBatch.length > 0) {
await processBatch(currentBatch);
}
if (jobId) {
await db.import_jobs.update(
{ status: 'completed', rows_processed: totalProcessed },
{ where: { id: jobId } }
);
}
console.log('Import finished. Total:', totalProcessed, 'Skipped:', totalSkipped);
// Send emails in background
if (emailsToInvite.length > 0 && sendInvitationEmails) {
(async () => {
const batchSize = 50;
for (let i = 0; i < emailsToInvite.length; i += batchSize) {
const batch = emailsToInvite.slice(i, i + batchSize);
await Promise.all(batch.map(email =>
AuthService.sendPasswordResetEmail(email, 'invitation', host).catch(err => console.error(`Email error for ${email}:`, err))
));
await new Promise(resolve => setTimeout(resolve, 1000));
}
})().catch(err => console.error('Background email error:', err));
}
return { totalProcessed, totalSkipped };
} catch (error) {
console.error('Bulk import error:', error);
if (jobId) {
await db.import_jobs.update(
{ status: 'failed' },
{ where: { id: jobId } }
).catch(e => console.error('Failed to update job status:', e));
}
throw error;
}
}

View File

@ -5,82 +5,87 @@ const menuAside: MenuAsideItem[] = [
{
href: '/dashboard',
icon: icon.mdiViewDashboardOutline,
label: 'Dashboard',
label: 'Tableau de bord',
},
{
href: '/check-in',
label: 'Pointage Entrée',
label: 'Enregistrement Entrée',
icon: icon.mdiClockIn,
permissions: 'CREATE_TIME_ENTRIES'
},
{
href: '/users/users-list',
label: 'Collaborateurs',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiAccountGroup ?? icon.mdiTable,
permissions: 'READ_USERS'
label: 'Données',
icon: icon.mdiDatabase,
menu: [
{
href: '/users/users-list',
label: 'Collaborateurs',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiAccountGroup ?? icon.mdiTable,
permissions: 'READ_USERS'
},
{
href: '/time_entries/time_entries-list',
label: 'Historique Entrées',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiClock' in icon ? icon['mdiClock' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_TIME_ENTRIES'
},
{
href: '/badges/badges-list',
label: 'Badges',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiBadgeAccount' in icon ? icon['mdiBadgeAccount' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_BADGES'
},
]
},
{
href: '/time_entries/time_entries-list',
label: 'Présences',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiClock' in icon ? icon['mdiClock' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_TIME_ENTRIES'
},
{
href: '/roles/roles-list',
label: 'Roles',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiShieldAccountVariantOutline ?? icon.mdiTable,
permissions: 'READ_ROLES'
},
{
href: '/permissions/permissions-list',
label: 'Permissions',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiShieldAccountOutline ?? icon.mdiTable,
permissions: 'READ_PERMISSIONS'
},
{
href: '/departments/departments-list',
label: 'Departments',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiOfficeBuilding' in icon ? icon['mdiOfficeBuilding' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_DEPARTMENTS'
},
{
href: '/import_jobs/import_jobs-list',
label: 'Import jobs',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiFileImport' in icon ? icon['mdiFileImport' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_IMPORT_JOBS'
},
{
href: '/badges/badges-list',
label: 'Badges',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiBadgeAccount' in icon ? icon['mdiBadgeAccount' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_BADGES'
label: 'Configuration',
icon: icon.mdiCog,
menu: [
{
href: '/import_jobs/import_jobs-list',
label: 'Imports CSV',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiFileImport' in icon ? icon['mdiFileImport' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_IMPORT_JOBS'
},
{
href: '/departments/departments-list',
label: 'Départements',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: 'mdiOfficeBuilding' in icon ? icon['mdiOfficeBuilding' as keyof typeof icon] : icon.mdiTable ?? icon.mdiTable,
permissions: 'READ_DEPARTMENTS'
},
{
href: '/roles/roles-list',
label: 'Rôles',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiShieldAccountVariantOutline ?? icon.mdiTable,
permissions: 'READ_ROLES'
},
{
href: '/permissions/permissions-list',
label: 'Permissions',
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
icon: icon.mdiShieldAccountOutline ?? icon.mdiTable,
permissions: 'READ_PERMISSIONS'
},
]
},
{
href: '/profile',
label: 'Profile',
label: 'Mon Profil',
icon: icon.mdiAccountCircle,
},
{
href: '/api-docs',
target: '_blank',
label: 'Swagger API',
icon: icon.mdiFileCode,
permissions: 'READ_API_DOCS'
},
]
export default menuAside
export default menuAside

View File

@ -2,7 +2,7 @@ import { mdiClockIn, mdiAccountSearch, mdiCalendarClock, mdiCardAccountDetailsOu
import Head from 'next/head'
import React, { ReactElement, useEffect, useState } from 'react'
import CardBox from '../components/CardBox'
import LayoutGuest from '../layouts/Guest'
import LayoutAuthenticated from '../layouts/Authenticated'
import SectionMain from '../components/SectionMain'
import SectionTitleLineWithButton from '../components/SectionTitleLineWithButton'
import { getPageTitle } from '../config'
@ -64,7 +64,7 @@ const EmployeeDetails = () => {
<div className="grid grid-cols-1 md:grid-cols-2 gap-4 text-sm">
<div>
<p className="text-gray-500">Nom & Prénom:</p>
<p className="font-semibold">{employeeData.firstName} {employeeData.lastName}</p>
<p className="font-semibold text-lg">{employeeData.firstName} {employeeData.lastName}</p>
</div>
<div>
<p className="text-gray-500">Département:</p>
@ -75,7 +75,7 @@ const EmployeeDetails = () => {
<p className="font-semibold">{employeeData.matriculePaie || '-'}</p>
</div>
<div>
<p className="text-gray-500">Workday ID:</p>
<p className="text-gray-500">WD ID:</p>
<p className="font-semibold">{employeeData.workdayId || '-'}</p>
</div>
<div>
@ -106,7 +106,7 @@ const EmployeeDetails = () => {
const CheckIn = () => {
const dispatch = useAppDispatch()
const [isSuccess, setIsSuccess] = useState(false)
const [error, setError] = useState(null)
const [error, setError] = useState<string | null>(null)
const handleSubmit = async (data, { resetForm }) => {
setIsSuccess(false)
@ -116,6 +116,8 @@ const CheckIn = () => {
if (create.fulfilled.match(resultAction)) {
setIsSuccess(true)
resetForm()
// Keep the start_time current
// data.start_time = moment().format('YYYY-MM-DDTHH:mm')
} else {
setError("Une erreur est survenue lors de l'enregistrement.")
}
@ -153,14 +155,14 @@ const CheckIn = () => {
>
{({ isSubmitting }) => (
<Form>
<FormField label="Collaborateur" labelFor="employee" icons={[mdiAccountSearch]}>
<FormField label="Rechercher un collaborateur (Nom)" labelFor="employee" icons={[mdiAccountSearch]}>
<Field
name="employee"
id="employee"
component={SelectField}
options={[]}
options={[]}
itemRef={'users'}
showField={'lastName'}
showField={'lastName'} // Not critical for autocomplete
></Field>
</FormField>
@ -168,13 +170,15 @@ const CheckIn = () => {
<BaseDivider />
<FormField label="Heure d'entrée" icons={[mdiCalendarClock]}>
<Field type="datetime-local" name="start_time" placeholder="Heure d'entrée" />
</FormField>
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
<FormField label="Date et Heure d'entrée" icons={[mdiCalendarClock]}>
<Field type="datetime-local" name="start_time" placeholder="Heure d'entrée" />
</FormField>
<FormField label="Numéro de badge provisoire" icons={[mdiCardAccountDetailsOutline]}>
<Field name="badge_number" placeholder="Ex: 12345" />
</FormField>
<FormField label="Numéro de badge provisoire" icons={[mdiCardAccountDetailsOutline]}>
<Field name="badge_number" placeholder="Ex: 12345" />
</FormField>
</div>
<BaseDivider />
@ -182,7 +186,7 @@ const CheckIn = () => {
<BaseButton
type="submit"
color="info"
label={isSubmitting ? "Enregistrement..." : "Enregistrer l'entrée"}
label={isSubmitting ? "Enregistrement..." : "Valider l'entrée"}
disabled={isSubmitting}
/>
<BaseButton
@ -204,10 +208,10 @@ const CheckIn = () => {
CheckIn.getLayout = function getLayout(page: ReactElement) {
return (
<LayoutGuest>
<LayoutAuthenticated>
{page}
</LayoutGuest>
</LayoutAuthenticated>
)
}
export default CheckIn
export default CheckIn