Revert to version fa68f42
This commit is contained in:
Flatlogic Bot
parent
8de95af8ac
commit
d67465d249
@ -12,6 +12,3 @@ EMAIL_USER=AKIAVEW7G4PQUBGM52OF
|
||||
EMAIL_PASS=BLnD4hKGb6YkSz3gaQrf8fnyLi3C3/EdjOOsLEDTDPTz
|
||||
SECRET_KEY=HUEyqESqgQ1yTwzVlO6wprC9Kf1J1xuA
|
||||
PEXELS_KEY=Vc99rnmOhHhJAbgGQoKLZtsaIVfkeownoQNbTj78VemUjKh08ZYRbf18
|
||||
FIRECRAWL_API_KEY=fc-409763513f6c458c9d1d09e460346b17
|
||||
FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2
|
||||
FIRECRAWL_ENABLED=true
|
||||
|
||||
@ -1,49 +1,4 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const axios = require('axios');
|
||||
|
||||
const FIRECRAWL_DEFAULT_BASE_URL = 'https://api.firecrawl.dev/v2';
|
||||
const FIRECRAWL_DEFAULT_POLL_INTERVAL_MS = 2000;
|
||||
const FIRECRAWL_DEFAULT_TIMEOUT_MS = 45000;
|
||||
|
||||
const BACKEND_ENV_PATH = path.join(__dirname, '..', '..', '.env');
|
||||
|
||||
function readBackendEnvFile() {
|
||||
try {
|
||||
const raw = fs.readFileSync(BACKEND_ENV_PATH, 'utf8');
|
||||
|
||||
return raw.split(/\r?\n/).reduce((accumulator, line) => {
|
||||
const trimmedLine = line.trim();
|
||||
|
||||
if (!trimmedLine || trimmedLine.startsWith('#')) {
|
||||
return accumulator;
|
||||
}
|
||||
|
||||
const separatorIndex = trimmedLine.indexOf('=');
|
||||
|
||||
if (separatorIndex === -1) {
|
||||
return accumulator;
|
||||
}
|
||||
|
||||
const key = trimmedLine.slice(0, separatorIndex).trim();
|
||||
const value = trimmedLine.slice(separatorIndex + 1).trim();
|
||||
|
||||
accumulator[key] = value.replace(/^"|"$/g, '').replace(/^'|'$/g, '');
|
||||
return accumulator;
|
||||
}, {});
|
||||
} catch (error) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function getEnvValue(name) {
|
||||
if (process.env[name] !== undefined && process.env[name] !== null && process.env[name] !== '') {
|
||||
return process.env[name];
|
||||
}
|
||||
|
||||
return readBackendEnvFile()[name];
|
||||
}
|
||||
|
||||
const FIRECRAWL_DEFAULT_BASE_URL = 'https://api.firecrawl.dev/v1';
|
||||
|
||||
function toBoolean(value, defaultValue = false) {
|
||||
if (value === undefined || value === null || value === '') {
|
||||
@ -67,34 +22,12 @@ function toBoolean(value, defaultValue = false) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
function toPositiveInteger(value, defaultValue) {
|
||||
const parsed = Number(value);
|
||||
|
||||
if (Number.isInteger(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(baseUrl) {
|
||||
return String(baseUrl || FIRECRAWL_DEFAULT_BASE_URL)
|
||||
.trim()
|
||||
.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
function getFirecrawlRuntime() {
|
||||
const apiKey = String(getEnvValue('FIRECRAWL_API_KEY') || '').trim();
|
||||
const baseUrl = normalizeBaseUrl(getEnvValue('FIRECRAWL_BASE_URL'));
|
||||
const enabled = toBoolean(getEnvValue('FIRECRAWL_ENABLED'), true);
|
||||
const pollIntervalMs = toPositiveInteger(
|
||||
getEnvValue('FIRECRAWL_POLL_INTERVAL_MS'),
|
||||
FIRECRAWL_DEFAULT_POLL_INTERVAL_MS,
|
||||
);
|
||||
const timeoutMs = toPositiveInteger(
|
||||
getEnvValue('FIRECRAWL_TIMEOUT_MS'),
|
||||
FIRECRAWL_DEFAULT_TIMEOUT_MS,
|
||||
);
|
||||
const apiKey = String(process.env.FIRECRAWL_API_KEY || '').trim();
|
||||
const baseUrl = String(
|
||||
process.env.FIRECRAWL_BASE_URL || FIRECRAWL_DEFAULT_BASE_URL,
|
||||
).trim();
|
||||
const enabled = toBoolean(process.env.FIRECRAWL_ENABLED, true);
|
||||
|
||||
return {
|
||||
provider: 'firecrawl',
|
||||
@ -102,255 +35,29 @@ function getFirecrawlRuntime() {
|
||||
enabled,
|
||||
configured: Boolean(apiKey),
|
||||
hasApiKey: Boolean(apiKey),
|
||||
apiKey,
|
||||
pollIntervalMs,
|
||||
timeoutMs,
|
||||
mode: enabled && apiKey ? 'active' : 'scaffold_only',
|
||||
mode: 'scaffold_only',
|
||||
};
|
||||
}
|
||||
|
||||
function buildFirecrawlMessage(runtime, entitlements, requestedPages) {
|
||||
if (!entitlements?.canAdvancedCrawl) {
|
||||
return 'Firecrawl is reserved for paid Advanced Crawl users. This request will stay on the built-in crawler.';
|
||||
}
|
||||
|
||||
if (!runtime.enabled) {
|
||||
return 'Firecrawl is configured in code, but FIRECRAWL_ENABLED is turned off. Paid users will stay on the built-in crawler until it is enabled.';
|
||||
}
|
||||
|
||||
if (!runtime.configured) {
|
||||
return 'Firecrawl is enabled for paid users, but FIRECRAWL_API_KEY is missing. Falling back to the built-in crawler until the key is configured.';
|
||||
}
|
||||
|
||||
return requestedPages > 1
|
||||
? 'Paid Advanced Crawl users are routed through Firecrawl for sitemap-aware, JavaScript-rendered multi-page crawling.'
|
||||
: 'Paid Advanced Crawl users are routed through Firecrawl for sitemap-aware, JavaScript-rendered crawling.';
|
||||
}
|
||||
|
||||
function getFirecrawlScaffold({ requestedPages, entitlements } = {}) {
|
||||
const runtime = getFirecrawlRuntime();
|
||||
const availableForCurrentUser = Boolean(entitlements?.canAdvancedCrawl);
|
||||
const shouldUseFirecrawl = Boolean(
|
||||
availableForCurrentUser
|
||||
&& runtime.enabled
|
||||
&& runtime.configured,
|
||||
);
|
||||
const wantsAdvancedCrawl = Number(requestedPages || 1) > 1;
|
||||
const advancedCrawlUnlocked = Boolean(entitlements?.canAdvancedCrawl);
|
||||
const shouldUseFirecrawlLater = runtime.enabled && (wantsAdvancedCrawl || advancedCrawlUnlocked);
|
||||
|
||||
return {
|
||||
provider: 'firecrawl',
|
||||
baseUrl: runtime.baseUrl,
|
||||
enabled: runtime.enabled,
|
||||
configured: runtime.configured,
|
||||
hasApiKey: runtime.hasApiKey,
|
||||
mode: shouldUseFirecrawl ? 'active' : runtime.mode,
|
||||
status: shouldUseFirecrawl ? 'active_for_paid_users' : 'scaffold_only',
|
||||
...runtime,
|
||||
status: runtime.configured ? 'ready_for_activation' : 'awaiting_api_key',
|
||||
wouldHandleJavascript: true,
|
||||
wouldHandleSitemapDiscovery: true,
|
||||
availableForCurrentUser,
|
||||
shouldUseFirecrawl,
|
||||
usePaidOnly: true,
|
||||
message: buildFirecrawlMessage(runtime, entitlements, requestedPages),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(milliseconds) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, milliseconds);
|
||||
});
|
||||
}
|
||||
|
||||
function isAbsoluteUrl(value) {
|
||||
return /^https?:\/\//i.test(String(value || ''));
|
||||
}
|
||||
|
||||
function buildApiUrl(runtime, pathOrUrl) {
|
||||
const value = String(pathOrUrl || '').trim();
|
||||
|
||||
if (!value) {
|
||||
return runtime.baseUrl;
|
||||
}
|
||||
|
||||
if (isAbsoluteUrl(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (value.startsWith('/')) {
|
||||
return `${runtime.baseUrl}${value}`;
|
||||
}
|
||||
|
||||
return `${runtime.baseUrl}/${value}`;
|
||||
}
|
||||
|
||||
function summarizeFirecrawlPayload(payload) {
|
||||
if (!payload) {
|
||||
return 'Unknown Firecrawl API error.';
|
||||
}
|
||||
|
||||
if (typeof payload === 'string') {
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (typeof payload?.error === 'string' && payload.error.trim()) {
|
||||
return payload.error;
|
||||
}
|
||||
|
||||
if (typeof payload?.message === 'string' && payload.message.trim()) {
|
||||
return payload.message;
|
||||
}
|
||||
|
||||
return 'Unexpected Firecrawl API response.';
|
||||
}
|
||||
|
||||
async function firecrawlRequest(runtime, method, pathOrUrl, options = {}) {
|
||||
try {
|
||||
const response = await axios({
|
||||
method,
|
||||
url: buildApiUrl(runtime, pathOrUrl),
|
||||
timeout: options.timeout || runtime.timeoutMs,
|
||||
data: options.data,
|
||||
headers: {
|
||||
Authorization: `Bearer ${runtime.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
...(options.headers || {}),
|
||||
},
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
const payload = error.response?.data;
|
||||
const detail = summarizeFirecrawlPayload(payload);
|
||||
const status = error.response?.status;
|
||||
const wrappedError = new Error(
|
||||
status
|
||||
? `Firecrawl request failed with status ${status}: ${detail}`
|
||||
: `Firecrawl request failed: ${detail}`,
|
||||
);
|
||||
|
||||
wrappedError.code = status || 502;
|
||||
wrappedError.response = payload;
|
||||
throw wrappedError;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectPagedStatus(runtime, initialStatus) {
|
||||
const documents = Array.isArray(initialStatus?.data)
|
||||
? [...initialStatus.data]
|
||||
: [];
|
||||
let nextUrl = initialStatus?.next || null;
|
||||
|
||||
while (nextUrl) {
|
||||
const nextStatus = await firecrawlRequest(runtime, 'get', nextUrl);
|
||||
|
||||
if (Array.isArray(nextStatus?.data) && nextStatus.data.length > 0) {
|
||||
documents.push(...nextStatus.data);
|
||||
}
|
||||
|
||||
nextUrl = nextStatus?.next || null;
|
||||
}
|
||||
|
||||
return {
|
||||
...initialStatus,
|
||||
data: documents,
|
||||
next: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForCrawlCompletion(runtime, crawlId) {
|
||||
const deadline = Date.now() + runtime.timeoutMs;
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
const status = await firecrawlRequest(runtime, 'get', `/crawl/${encodeURIComponent(crawlId)}`);
|
||||
|
||||
if (status?.status === 'completed' || status?.status === 'failed') {
|
||||
return collectPagedStatus(runtime, status);
|
||||
}
|
||||
|
||||
await sleep(runtime.pollIntervalMs);
|
||||
}
|
||||
|
||||
const timeoutError = new Error(
|
||||
`Firecrawl crawl timed out after ${Math.round(runtime.timeoutMs / 1000)} seconds.`,
|
||||
);
|
||||
timeoutError.code = 504;
|
||||
throw timeoutError;
|
||||
}
|
||||
|
||||
async function getCrawlErrors(runtime, crawlId) {
|
||||
try {
|
||||
return await firecrawlRequest(runtime, 'get', `/crawl/${encodeURIComponent(crawlId)}/errors`);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch Firecrawl crawl errors:', error);
|
||||
return {
|
||||
errors: [],
|
||||
robotsBlocked: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function crawlSiteWithFirecrawl(url, requestedPages) {
|
||||
const runtime = getFirecrawlRuntime();
|
||||
|
||||
if (!runtime.enabled) {
|
||||
const error = new Error('Firecrawl is disabled in this environment.');
|
||||
error.code = 503;
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!runtime.configured) {
|
||||
const error = new Error('Firecrawl API key is not configured.');
|
||||
error.code = 503;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const started = await firecrawlRequest(runtime, 'post', '/crawl', {
|
||||
data: {
|
||||
url,
|
||||
limit: requestedPages,
|
||||
sitemap: 'include',
|
||||
crawlEntireDomain: true,
|
||||
allowExternalLinks: false,
|
||||
allowSubdomains: false,
|
||||
ignoreQueryParameters: true,
|
||||
scrapeOptions: {
|
||||
formats: ['html'],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const crawlId = started?.id;
|
||||
|
||||
if (!crawlId) {
|
||||
const error = new Error('Firecrawl did not return a crawl job ID.');
|
||||
error.code = 502;
|
||||
error.response = started;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const status = await waitForCrawlCompletion(runtime, crawlId);
|
||||
const crawlErrors = await getCrawlErrors(runtime, crawlId);
|
||||
|
||||
return {
|
||||
crawlId,
|
||||
provider: 'firecrawl',
|
||||
status: status?.status || 'unknown',
|
||||
total: status?.total || 0,
|
||||
completed: status?.completed || 0,
|
||||
creditsUsed: status?.creditsUsed || 0,
|
||||
expiresAt: status?.expiresAt || null,
|
||||
data: Array.isArray(status?.data) ? status.data : [],
|
||||
errors: Array.isArray(crawlErrors?.errors) ? crawlErrors.errors : [],
|
||||
robotsBlocked: Array.isArray(crawlErrors?.robotsBlocked)
|
||||
? crawlErrors.robotsBlocked
|
||||
: [],
|
||||
shouldUseFirecrawlLater,
|
||||
message: runtime.configured
|
||||
? 'Firecrawl scaffold is wired and ready for the next activation step, but this analyzer still uses the built-in crawler today.'
|
||||
: 'Firecrawl scaffold is wired, but FIRECRAWL_API_KEY is not set yet. The analyzer still uses the built-in crawler for now.',
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getFirecrawlRuntime,
|
||||
getFirecrawlScaffold,
|
||||
crawlSiteWithFirecrawl,
|
||||
};
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
const ValidationError = require('./notifications/errors/validation');
|
||||
|
||||
const BASIC_MAX_PAGES_PER_CRAWL = 25;
|
||||
const BASIC_MAX_PAGES_PER_CRAWL = 1;
|
||||
const ADVANCED_MAX_PAGES_PER_CRAWL = 25;
|
||||
const ADVANCED_CRAWL_PERMISSION = 'USE_ADVANCED_CRAWL';
|
||||
const PLATFORM_OUTPUT_PERMISSION = 'USE_PLATFORM_OUTPUT';
|
||||
@ -68,7 +68,7 @@ function ensureRequestedPagesAllowed(requestedPages, currentUser) {
|
||||
|
||||
if (requestedPages > entitlements.maxPagesPerCrawl) {
|
||||
const error = new Error(
|
||||
`This analyzer supports up to ${entitlements.maxPagesPerCrawl} page${entitlements.maxPagesPerCrawl === 1 ? '' : 's'} per crawl. Reduce the requested page count to continue.`,
|
||||
`Your current plan allows up to ${entitlements.maxPagesPerCrawl} page${entitlements.maxPagesPerCrawl === 1 ? '' : 's'} per crawl. Upgrade to Advanced Crawl to analyze ${requestedPages} pages.`,
|
||||
);
|
||||
error.code = 403;
|
||||
throw error;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,4 +1,3 @@
|
||||
import Image from 'next/image'
|
||||
import React from 'react'
|
||||
|
||||
type Props = {
|
||||
@ -7,12 +6,10 @@ type Props = {
|
||||
|
||||
export default function Logo({ className = '' }: Props) {
|
||||
return (
|
||||
<Image
|
||||
src="https://flatlogic.com/logo.svg"
|
||||
<img
|
||||
src={"https://flatlogic.com/logo.svg"}
|
||||
className={className}
|
||||
alt="Flatlogic logo"
|
||||
width={160}
|
||||
height={32}
|
||||
/>
|
||||
alt={'Flatlogic logo'}>
|
||||
</img>
|
||||
)
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { hasPermission } from './userPermissions';
|
||||
|
||||
export const BASIC_MAX_PAGES_PER_CRAWL = 25;
|
||||
export const BASIC_MAX_PAGES_PER_CRAWL = 1;
|
||||
export const ADVANCED_MAX_PAGES_PER_CRAWL = 25;
|
||||
export const ADVANCED_CRAWL_PERMISSION = 'USE_ADVANCED_CRAWL';
|
||||
export const PLATFORM_OUTPUT_PERMISSION = 'USE_PLATFORM_OUTPUT';
|
||||
|
||||
@ -3,9 +3,10 @@ import {
|
||||
mdiUpload,
|
||||
} from '@mdi/js';
|
||||
import Head from 'next/head';
|
||||
import Image from 'next/image';
|
||||
import React, { ReactElement, useEffect, useState } from 'react';
|
||||
import { toast } from 'react-toastify';
|
||||
import { ToastContainer, toast } from 'react-toastify';
|
||||
import DatePicker from 'react-datepicker';
|
||||
import 'react-datepicker/dist/react-datepicker.css';
|
||||
|
||||
import CardBox from '../components/CardBox';
|
||||
import LayoutAuthenticated from '../layouts/Authenticated';
|
||||
@ -18,17 +19,21 @@ import FormField from '../components/FormField';
|
||||
import BaseDivider from '../components/BaseDivider';
|
||||
import BaseButtons from '../components/BaseButtons';
|
||||
import BaseButton from '../components/BaseButton';
|
||||
import FormCheckRadio from '../components/FormCheckRadio';
|
||||
import FormCheckRadioGroup from '../components/FormCheckRadioGroup';
|
||||
import FormImagePicker from '../components/FormImagePicker';
|
||||
import { SwitchField } from '../components/SwitchField';
|
||||
import { SelectField } from '../components/SelectField';
|
||||
|
||||
import { update } from '../stores/users/usersSlice';
|
||||
import { update, fetch } from '../stores/users/usersSlice';
|
||||
import { useAppDispatch, useAppSelector } from '../stores/hooks';
|
||||
import { useRouter } from 'next/router';
|
||||
import { findMe } from '../stores/authSlice';
|
||||
import {findMe} from "../stores/authSlice";
|
||||
|
||||
const EditUsers = () => {
|
||||
const { currentUser } = useAppSelector((state) => state.auth);
|
||||
const { currentUser, isFetching, token } = useAppSelector(
|
||||
(state) => state.auth,
|
||||
);
|
||||
const router = useRouter();
|
||||
const dispatch = useAppDispatch();
|
||||
const notify = (type, msg) => toast(msg, { type });
|
||||
@ -77,15 +82,9 @@ const EditUsers = () => {
|
||||
{''}
|
||||
</SectionTitleLineWithButton>
|
||||
<CardBox>
|
||||
{currentUser?.avatar?.[0]?.publicUrl && <div className={'grid grid-cols-6 gap-4 mb-4'}>
|
||||
{currentUser?.avatar[0]?.publicUrl && <div className={'grid grid-cols-6 gap-4 mb-4'}>
|
||||
<div className="col-span-1 w-80 h-80 overflow-hidden border-2 rounded-full inline-flex items-center justify-center mb-8">
|
||||
<Image
|
||||
className="w-80 h-80 max-w-full max-h-full object-cover object-center"
|
||||
src={`${currentUser?.avatar?.[0]?.publicUrl}`}
|
||||
alt="Avatar"
|
||||
width={320}
|
||||
height={320}
|
||||
/>
|
||||
<img className="w-80 h-80 max-w-full max-h-full object-cover object-center" src={`${currentUser?.avatar[0]?.publicUrl}`} alt="Avatar" />
|
||||
</div>
|
||||
</div>}
|
||||
<Formik
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user