From 8b29dd2b92e98f840c6ae62a4bb2d1ccd74d5e5f Mon Sep 17 00:00:00 2001 From: Dmitri Date: Thu, 25 Jun 2026 08:38:21 +0200 Subject: [PATCH] added guard for hevy videos processing (ffmpeg reverse video creation) --- backend/src/db/api/assets.js | 3 +- ...20260625000001-add-frame-rate-to-assets.js | 14 ++ backend/src/db/models/assets.js | 4 + backend/src/routes/assets.js | 2 + backend/src/services/assets.js | 60 +++++- backend/src/services/file.js | 45 +++++ backend/src/services/projects.js | 1 + backend/src/services/tour_pages.js | 178 +++++++++++++++++- backend/src/services/videoProcessing.js | 66 +++++++ frontend/src/schemas/assetSchema.ts | 6 + frontend/src/types/entities.ts | 1 + 11 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 backend/src/db/migrations/20260625000001-add-frame-rate-to-assets.js diff --git a/backend/src/db/api/assets.js b/backend/src/db/api/assets.js index 54da223..639b2e3 100644 --- a/backend/src/db/api/assets.js +++ b/backend/src/db/api/assets.js @@ -22,7 +22,7 @@ class AssetsDBApi extends GenericDBApi { } static get RANGE_FIELDS() { - return ['size_mb', 'width_px', 'height_px', 'duration_sec']; + return ['size_mb', 'width_px', 'height_px', 'duration_sec', 'frame_rate']; } static get ENUM_FIELDS() { @@ -90,6 +90,7 @@ class AssetsDBApi extends GenericDBApi { width_px: data.width_px || null, height_px: data.height_px || null, duration_sec: data.duration_sec || null, + frame_rate: data.frame_rate || null, embed_code: data.embed_code || null, embed_provider: data.embed_provider || null, checksum: data.checksum || null, diff --git a/backend/src/db/migrations/20260625000001-add-frame-rate-to-assets.js b/backend/src/db/migrations/20260625000001-add-frame-rate-to-assets.js new file mode 100644 index 0000000..ba1dc2e --- /dev/null +++ b/backend/src/db/migrations/20260625000001-add-frame-rate-to-assets.js @@ -0,0 +1,14 @@ +'use strict'; + +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('assets', 'frame_rate', { + type: Sequelize.DECIMAL, + allowNull: true, + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn('assets', 'frame_rate'); + }, +}; diff --git a/backend/src/db/models/assets.js b/backend/src/db/models/assets.js index 6ec0c44..08ded09 100644 --- a/backend/src/db/models/assets.js +++ b/backend/src/db/models/assets.js @@ -91,6 +91,10 @@ module.exports = function (sequelize, DataTypes) { type: DataTypes.DECIMAL, }, + frame_rate: { + type: DataTypes.DECIMAL, + }, + embed_code: { type: DataTypes.TEXT, allowNull: true, diff --git a/backend/src/routes/assets.js b/backend/src/routes/assets.js index e25e2af..efe2d67 100644 --- a/backend/src/routes/assets.js +++ b/backend/src/routes/assets.js @@ -27,6 +27,8 @@ const { createEntityRouter } = require('../factories/router.factory'); * type: number * duration_sec: * type: number + * frame_rate: + * type: number */ /** diff --git a/backend/src/services/assets.js b/backend/src/services/assets.js index 8d8cfae..865f0e6 100644 --- a/backend/src/services/assets.js +++ b/backend/src/services/assets.js @@ -1,6 +1,9 @@ const AssetsDBApi = require('../db/api/assets'); const { createEntityService } = require('../factories/service.factory'); const ValidationError = require('./notifications/errors/validation'); +const { downloadToTempFile } = require('./file'); +const { probeMediaMetadata } = require('./videoProcessing'); +const { logger } = require('../utils/logger'); // Note: Reversed video generation was moved to tour_pages.js to generate // only when videos are actually used as transitions (not all video uploads). @@ -146,6 +149,50 @@ const BaseService = createEntityService(AssetsDBApi, { * Assets Service with validation and video pre-processing */ class AssetsService extends BaseService { + static async enrichStoredMediaMetadata(data) { + const assetType = data.asset_type; + const storageKey = data.storage_key; + + if (!storageKey) return data; + if (assetType !== 'video' && assetType !== 'audio') { + return data; + } + + let tempFile = null; + + try { + tempFile = await downloadToTempFile(storageKey); + const metadata = await probeMediaMetadata(tempFile.filePath); + + if (!metadata) { + return data; + } + + return { + ...data, + duration_sec: metadata.durationSec ?? data.duration_sec ?? null, + width_px: metadata.widthPx ?? data.width_px ?? null, + height_px: metadata.heightPx ?? data.height_px ?? null, + frame_rate: metadata.frameRate ?? data.frame_rate ?? null, + }; + } catch (error) { + logger.warn( + { err: error, storageKey, assetType }, + 'Failed to probe stored media metadata', + ); + return data; + } finally { + if (tempFile?.cleanup) { + await tempFile.cleanup().catch((cleanupError) => { + logger.warn( + { err: cleanupError, storageKey }, + 'Failed to cleanup media probe temp file', + ); + }); + } + } + } + /** * Create asset with MIME type validation, embed URL validation, and video pre-processing */ @@ -166,6 +213,8 @@ class AssetsService extends BaseService { data.embed_provider = provider; } + data = await AssetsService.enrichStoredMediaMetadata(data); + // Call parent create const asset = await super.create(data, currentUser); @@ -180,10 +229,12 @@ class AssetsService extends BaseService { * Update asset with MIME type validation and embed URL validation */ static async update(data, id, currentUser) { + const existingAsset = await AssetsDBApi.findBy({ id }); + // If updating asset_type or mime_type, validate they match if (data.asset_type || data.mime_type) { - const assetType = data.asset_type; - const mimeType = data.mime_type; + const assetType = data.asset_type || existingAsset?.asset_type; + const mimeType = data.mime_type || existingAsset?.mime_type; // Only validate if both are provided in the update if (assetType && mimeType) { @@ -201,6 +252,11 @@ class AssetsService extends BaseService { data.embed_provider = provider; } + data = await AssetsService.enrichStoredMediaMetadata({ + ...existingAsset, + ...data, + }); + // Call parent update return super.update(data, id, currentUser); } diff --git a/backend/src/services/file.js b/backend/src/services/file.js index a2cc157..ff5c172 100644 --- a/backend/src/services/file.js +++ b/backend/src/services/file.js @@ -14,6 +14,7 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); +const os = require('os'); const { pipeline } = require('stream/promises'); const { format } = require('util'); @@ -825,6 +826,49 @@ const downloadToBuffer = async (privateUrl) => { } }; +/** + * Download a file to a temporary local path for probing/processing. + * Returns a cleanup function that removes the temp directory. + * + * @param {string} privateUrl - Storage key/path + * @returns {Promise<{filePath: string, cleanup: () => Promise}>} + */ +const downloadToTempFile = async (privateUrl) => { + const provider = getFileStorageProvider(); + const tempDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'asset-probe-'), + ); + const ext = path.extname(privateUrl) || ''; + const filePath = path.join(tempDir, `source${ext}`); + + const cleanup = async () => { + await fs.promises.rm(tempDir, { recursive: true, force: true }); + }; + + try { + if (provider === 's3') { + const s3 = getS3Provider(); + const result = await s3.download(privateUrl); + if (!result?.body) { + throw new Error(`Empty S3 response body for ${privateUrl}`); + } + await pipeline(result.body, fs.createWriteStream(filePath)); + } else if (provider === 'gcloud') { + const { bucket, hash } = getGCloudBucket(); + const file = bucket.file(`${hash}/${privateUrl}`); + await file.download({ destination: filePath }); + } else { + const local = getLocalProvider(); + await fs.promises.copyFile(local.buildPath(privateUrl), filePath); + } + + return { filePath, cleanup }; + } catch (error) { + await cleanup().catch(() => {}); + throw error; + } +}; + /** * Upload buffer to storage * @param {string} privateUrl - Storage key/path @@ -1309,6 +1353,7 @@ module.exports = { deleteFile, // Buffer operations downloadToBuffer, + downloadToTempFile, uploadBuffer, // File copy utilities copyFile, diff --git a/backend/src/services/projects.js b/backend/src/services/projects.js index 5376642..a3562fb 100644 --- a/backend/src/services/projects.js +++ b/backend/src/services/projects.js @@ -333,6 +333,7 @@ class ProjectsService extends BaseProjectsService { width_px: sourceAsset.width_px, height_px: sourceAsset.height_px, duration_sec: sourceAsset.duration_sec, + frame_rate: sourceAsset.frame_rate, checksum: sourceAsset.checksum, is_public: sourceAsset.is_public, projectId: clonedProject.id, diff --git a/backend/src/services/tour_pages.js b/backend/src/services/tour_pages.js index 3672e5c..4b4e60c 100644 --- a/backend/src/services/tour_pages.js +++ b/backend/src/services/tour_pages.js @@ -9,7 +9,8 @@ const Tour_pagesDBApi = require('../db/api/tour_pages'); const AssetsDBApi = require('../db/api/assets'); const Asset_variantsDBApi = require('../db/api/asset_variants'); const { createEntityService } = require('../factories/service.factory'); -const { downloadToBuffer, uploadBuffer } = require('./file'); +const { downloadToBuffer, downloadToTempFile, uploadBuffer } = require('./file'); +const ValidationError = require('./notifications/errors/validation'); const videoProcessing = require('./videoProcessing'); const { logger } = require('../utils/logger'); const db = require('../db/models'); @@ -17,6 +18,10 @@ const db = require('../db/models'); const projectRegenInProgress = new Set(); const singleReverseGenerationInProgress = new Set(); const reverseGenerationPromiseByStorageKey = new Map(); +const MAX_AUTO_REVERSE_SOURCE_SIZE_BYTES = 16 * 1024 * 1024 * 1024; +const DEFAULT_AUTO_REVERSE_FALLBACK_FPS = 30; +const YUV420_BYTES_PER_PIXEL = 1.5; +const MAX_AUTO_REVERSE_ESTIMATED_DECODED_BYTES = 2 * 1024 * 1024 * 1024; // Create base service from factory const BaseService = createEntityService(Tour_pagesDBApi, { @@ -27,6 +32,153 @@ const BaseService = createEntityService(Tour_pagesDBApi, { * Tour Pages Service with reversed video generation */ class TourPagesService extends BaseService { + static formatBytesToGiB(bytes) { + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`; + } + + static getAssetSizeBytes(asset) { + if (!asset || asset.size_mb == null) return null; + + const sizeMb = Number(asset.size_mb); + if (!Number.isFinite(sizeMb) || sizeMb < 0) return null; + + return Math.round(sizeMb * 1024 * 1024); + } + + static getAssetVideoMetadata(asset) { + const widthPx = Number(asset?.width_px); + const heightPx = Number(asset?.height_px); + const durationSec = Number(asset?.duration_sec); + const frameRate = Number(asset?.frame_rate); + + return { + widthPx: Number.isFinite(widthPx) && widthPx > 0 ? widthPx : null, + heightPx: Number.isFinite(heightPx) && heightPx > 0 ? heightPx : null, + durationSec: + Number.isFinite(durationSec) && durationSec > 0 ? durationSec : null, + frameRate: + Number.isFinite(frameRate) && frameRate > 0 ? frameRate : null, + }; + } + + static getEstimatedDecodedBytes({ widthPx, heightPx, durationSec, fps }) { + if (!widthPx || !heightPx || !durationSec || !fps) { + return null; + } + + return Math.round( + widthPx * + heightPx * + YUV420_BYTES_PER_PIXEL * + durationSec * + fps, + ); + } + + static formatBytesToMiB(bytes) { + return `${(bytes / (1024 * 1024)).toFixed(0)} MiB`; + } + + static async resolveAssetFrameRate(asset) { + const metadata = TourPagesService.getAssetVideoMetadata(asset); + if (metadata.frameRate) { + return metadata.frameRate; + } + + if (asset?.asset_type !== 'video' || !asset?.storage_key) { + return null; + } + + let tempFile = null; + + try { + tempFile = await downloadToTempFile(asset.storage_key); + const probedMetadata = await videoProcessing.probeMediaMetadata( + tempFile.filePath, + ); + const probedFrameRate = Number(probedMetadata?.frameRate); + + return Number.isFinite(probedFrameRate) && probedFrameRate > 0 + ? probedFrameRate + : null; + } catch (error) { + logger.warn( + { err: error, storageKey: asset.storage_key }, + 'Failed to probe asset frame rate during auto-reverse validation', + ); + return null; + } finally { + if (tempFile?.cleanup) { + await tempFile.cleanup().catch((cleanupError) => { + logger.warn( + { err: cleanupError, storageKey: asset?.storage_key }, + 'Failed to cleanup frame-rate probe temp file', + ); + }); + } + } + } + + static async validateAutoReverseAssets(storageKeys) { + for (const storageKey of storageKeys) { + const asset = await AssetsDBApi.findBy({ storage_key: storageKey }); + + if (!asset) { + logger.warn({ storageKey }, 'Asset not found during auto-reverse validation'); + continue; + } + + const variants = asset.asset_variants_asset || []; + const hasExistingReversedVariant = variants.some( + (variant) => variant.variant_type === 'reversed', + ); + if (hasExistingReversedVariant) { + continue; + } + + const assetLabel = + asset.original_file_name || asset.name || asset.storage_key || 'video'; + + const sizeBytes = TourPagesService.getAssetSizeBytes(asset); + const { widthPx, heightPx, durationSec } = + TourPagesService.getAssetVideoMetadata(asset); + const frameRate = await TourPagesService.resolveAssetFrameRate(asset); + const effectiveFrameRate = + frameRate || DEFAULT_AUTO_REVERSE_FALLBACK_FPS; + const estimatedDecodedBytes = TourPagesService.getEstimatedDecodedBytes({ + widthPx, + heightPx, + durationSec, + fps: effectiveFrameRate, + }); + + if ( + sizeBytes != null && + sizeBytes > MAX_AUTO_REVERSE_SOURCE_SIZE_BYTES + ) { + throw new ValidationError( + `Transition video "${assetLabel}" is ${TourPagesService.formatBytesToGiB(sizeBytes)}. Auto-reverse is limited to 16 GiB source files. Use a smaller video or switch reverse mode to separate_video.`, + ); + } + + if ( + estimatedDecodedBytes != null && + estimatedDecodedBytes > MAX_AUTO_REVERSE_ESTIMATED_DECODED_BYTES + ) { + const dimensionLabel = + widthPx && heightPx ? `${widthPx}x${heightPx}` : 'unknown resolution'; + const durationLabel = durationSec ? `${durationSec.toFixed(2)}s` : 'unknown duration'; + const fpsLabel = frameRate + ? `${frameRate.toFixed(3)} FPS` + : `${DEFAULT_AUTO_REVERSE_FALLBACK_FPS} FPS fallback`; + + throw new ValidationError( + `Transition video "${assetLabel}" is too heavy for auto-reverse on this VM: ${dimensionLabel}, ${durationLabel}, ${fpsLabel} => about ${TourPagesService.formatBytesToMiB(estimatedDecodedBytes)} decoded video data. Use a shorter/smaller video or switch reverse mode to separate_video.`, + ); + } + } + } + static async reorder(data, currentUser) { const projectId = data?.projectId || data?.project; const environment = data?.environment || 'dev'; @@ -186,6 +338,7 @@ class TourPagesService extends BaseService { 'Processing reversed videos for navigation elements', ); + const storageKeysToValidate = new Set(); let wasModified = false; for (const element of uiSchema.elements) { @@ -221,6 +374,29 @@ class TourPagesService extends BaseService { } const storageKey = element.transitionVideoUrl; + if (element.transitionReverseMode !== 'separate_video') { + storageKeysToValidate.add(storageKey); + } + } + + if (storageKeysToValidate.size > 0) { + await TourPagesService.validateAutoReverseAssets(storageKeysToValidate); + } + + for (const element of uiSchema.elements) { + const isBack = TourPagesService.isBackElement(element); + const isForward = TourPagesService.isForwardElementWithTarget(element); + const needsReversed = isBack || isForward; + + if (!needsReversed) continue; + if (!element.transitionVideoUrl) continue; + const storageKey = element.transitionVideoUrl; + if ( + element.transitionReverseMode === 'separate_video' && + element.reverseVideoUrl + ) { + continue; + } try { // Fast path: check if reversed variant already exists diff --git a/backend/src/services/videoProcessing.js b/backend/src/services/videoProcessing.js index cc91497..b85cc50 100644 --- a/backend/src/services/videoProcessing.js +++ b/backend/src/services/videoProcessing.js @@ -21,6 +21,32 @@ let ffmpegQueueTail = Promise.resolve(); let queuedFfmpegJobs = 0; let ffmpegJobSequence = 0; +function parseFrameRate(value) { + if (!value) return null; + + if (typeof value === 'number') { + return Number.isFinite(value) && value > 0 ? value : null; + } + + const normalized = String(value).trim(); + if (!normalized || normalized === '0/0') return null; + + if (normalized.includes('/')) { + const [numeratorRaw, denominatorRaw] = normalized.split('/'); + const numerator = Number(numeratorRaw); + const denominator = Number(denominatorRaw); + if (!Number.isFinite(numerator) || !Number.isFinite(denominator)) { + return null; + } + if (denominator === 0) return null; + const fps = numerator / denominator; + return Number.isFinite(fps) && fps > 0 ? fps : null; + } + + const parsed = Number(normalized); + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; +} + async function enqueueFfmpegJob(jobName, runJob) { const jobId = ++ffmpegJobSequence; const queuedAhead = queuedFfmpegJobs; @@ -129,6 +155,45 @@ async function reverseVideoWithoutQueue(inputBuffer, filename) { } } +async function probeMediaMetadata(filePath) { + return new Promise((resolve, reject) => { + ffmpeg.ffprobe(filePath, (err, metadata) => { + if (err) { + reject(err); + return; + } + + const videoStream = metadata?.streams?.find( + (stream) => stream.codec_type === 'video', + ); + const audioStream = metadata?.streams?.find( + (stream) => stream.codec_type === 'audio', + ); + const primaryStream = videoStream || audioStream || null; + const formatDuration = Number(metadata?.format?.duration); + const streamDuration = Number(primaryStream?.duration); + const durationSec = Number.isFinite(formatDuration) && formatDuration > 0 + ? formatDuration + : Number.isFinite(streamDuration) && streamDuration > 0 + ? streamDuration + : null; + + const widthPx = Number(videoStream?.width); + const heightPx = Number(videoStream?.height); + const frameRate = + parseFrameRate(videoStream?.avg_frame_rate) || + parseFrameRate(videoStream?.r_frame_rate); + + resolve({ + durationSec, + widthPx: Number.isFinite(widthPx) && widthPx > 0 ? widthPx : null, + heightPx: Number.isFinite(heightPx) && heightPx > 0 ? heightPx : null, + frameRate, + }); + }); + }); +} + /** * Check if FFmpeg is available * @returns {Promise} @@ -144,4 +209,5 @@ async function isFFmpegAvailable() { module.exports = { reverseVideo, isFFmpegAvailable, + probeMediaMetadata, }; diff --git a/frontend/src/schemas/assetSchema.ts b/frontend/src/schemas/assetSchema.ts index ab79e6a..c2a7571 100644 --- a/frontend/src/schemas/assetSchema.ts +++ b/frontend/src/schemas/assetSchema.ts @@ -57,6 +57,11 @@ export const assetSchema = z.object({ .min(0, 'Duration cannot be negative') .optional() .or(z.literal('')), + frame_rate: z.coerce + .number() + .min(0, 'Frame rate cannot be negative') + .optional() + .or(z.literal('')), checksum: z .string() .max(255, 'Checksum too long') @@ -83,6 +88,7 @@ export const assetInitialValues: AssetFormData = { width_px: '', height_px: '', duration_sec: '', + frame_rate: '', checksum: '', embed_code: '', embed_provider: '', diff --git a/frontend/src/types/entities.ts b/frontend/src/types/entities.ts index 836c272..43a5620 100644 --- a/frontend/src/types/entities.ts +++ b/frontend/src/types/entities.ts @@ -80,6 +80,7 @@ export interface Asset extends BaseEntity { width_px?: number; height_px?: number; duration_sec?: number; + frame_rate?: number; embed_code?: string; embed_provider?: string; checksum?: string;