cleanup: remove unused link preview and goose response form components (#4795)
This commit is contained in:
Zane
@@ -1,8 +1,5 @@
|
|||||||
import { useEffect, useMemo, useRef } from 'react';
|
import { useEffect, useMemo, useRef } from 'react';
|
||||||
import LinkPreview from './LinkPreview';
|
|
||||||
import ImagePreview from './ImagePreview';
|
import ImagePreview from './ImagePreview';
|
||||||
import GooseResponseForm from './GooseResponseForm';
|
|
||||||
import { extractUrls } from '../utils/urlUtils';
|
|
||||||
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
|
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
|
||||||
import { formatMessageTimestamp } from '../utils/timeUtils';
|
import { formatMessageTimestamp } from '../utils/timeUtils';
|
||||||
import MarkdownContent from './MarkdownContent';
|
import MarkdownContent from './MarkdownContent';
|
||||||
@@ -44,7 +41,6 @@ export default function GooseMessage({
|
|||||||
sessionId,
|
sessionId,
|
||||||
messageHistoryIndex,
|
messageHistoryIndex,
|
||||||
message,
|
message,
|
||||||
metadata,
|
|
||||||
messages,
|
messages,
|
||||||
toolCallNotifications,
|
toolCallNotifications,
|
||||||
append,
|
append,
|
||||||
@@ -135,15 +131,6 @@ export default function GooseMessage({
|
|||||||
|
|
||||||
// Get the chain this message belongs to
|
// Get the chain this message belongs to
|
||||||
const messageChain = getChainForMessage(messageIndex, toolCallChains);
|
const messageChain = getChainForMessage(messageIndex, toolCallChains);
|
||||||
|
|
||||||
// Extract URLs under a few conditions
|
|
||||||
// 1. The message is purely text
|
|
||||||
// 2. The link wasn't also present in the previous message
|
|
||||||
// 3. The message contains the explicit http:// or https:// protocol at the beginning
|
|
||||||
const previousMessage = messageIndex > 0 ? messages[messageIndex - 1] : null;
|
|
||||||
const previousUrls = previousMessage ? extractUrls(getTextContent(previousMessage)) : [];
|
|
||||||
const urls = toolRequests.length === 0 ? extractUrls(displayText, previousUrls) : [];
|
|
||||||
|
|
||||||
const toolConfirmationContent = getToolConfirmationContent(message);
|
const toolConfirmationContent = getToolConfirmationContent(message);
|
||||||
const hasToolConfirmation = toolConfirmationContent !== undefined;
|
const hasToolConfirmation = toolConfirmationContent !== undefined;
|
||||||
|
|
||||||
@@ -302,27 +289,6 @@ export default function GooseMessage({
|
|||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* TODO(alexhancock): Re-enable link previews once styled well again */}
|
|
||||||
{/* TEMPORARILY DISABLED (dorien-koelemeijer): This is causing issues in properly "generating" tool calls
|
|
||||||
that contain links and prevents security scanning */}
|
|
||||||
{/* eslint-disable-next-line no-constant-binary-expression */}
|
|
||||||
{false && urls.length > 0 && (
|
|
||||||
<div className="mt-4">
|
|
||||||
{urls.map((url, index) => (
|
|
||||||
<LinkPreview key={index} url={url} />
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* enable or disable prompts here */}
|
|
||||||
{/* NOTE from alexhancock on 1/14/2025 - disabling again temporarily due to non-determinism in when the forms show up */}
|
|
||||||
{/* eslint-disable-next-line no-constant-binary-expression */}
|
|
||||||
{false && metadata && (
|
|
||||||
<div className="flex mt-[16px]">
|
|
||||||
<GooseResponseForm message={displayText} metadata={metadata || null} append={append} />
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,237 +0,0 @@
|
|||||||
import React, { useState, useEffect, useRef } from 'react';
|
|
||||||
import MarkdownContent from './MarkdownContent';
|
|
||||||
import { Button } from './ui/button';
|
|
||||||
import { cn } from '../utils';
|
|
||||||
import { Send } from './icons';
|
|
||||||
// Prefixing unused imports with underscore
|
|
||||||
import { createUserMessage as _createUserMessage } from '../types/message';
|
|
||||||
|
|
||||||
interface FormField {
|
|
||||||
label: string;
|
|
||||||
type: 'text' | 'textarea';
|
|
||||||
name: string;
|
|
||||||
placeholder: string;
|
|
||||||
required: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DynamicForm {
|
|
||||||
title: string;
|
|
||||||
description: string;
|
|
||||||
fields: FormField[];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface GooseResponseFormProps {
|
|
||||||
message: string;
|
|
||||||
metadata: string[] | null;
|
|
||||||
append: (value: string) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function GooseResponseForm({
|
|
||||||
message: _message,
|
|
||||||
metadata,
|
|
||||||
append,
|
|
||||||
}: GooseResponseFormProps) {
|
|
||||||
const [selectedOption, setSelectedOption] = useState<number | null>(null);
|
|
||||||
const [formValues, setFormValues] = useState<Record<string, string>>({});
|
|
||||||
const prevStatusRef = useRef<string | null>(null);
|
|
||||||
|
|
||||||
let isQuestion = false;
|
|
||||||
let isOptions = false;
|
|
||||||
let options: Array<{ optionTitle: string; optionDescription: string }> = [];
|
|
||||||
let dynamicForm: DynamicForm | null = null;
|
|
||||||
|
|
||||||
if (metadata) {
|
|
||||||
window.electron.logInfo('metadata:' + JSON.stringify(metadata, null, 2));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process metadata outside of conditional
|
|
||||||
const currentStatus = metadata?.[0] ?? null;
|
|
||||||
isQuestion = currentStatus === 'QUESTION';
|
|
||||||
isOptions = metadata?.[1] === 'OPTIONS';
|
|
||||||
|
|
||||||
// Parse dynamic form data if it exists in metadata[3]
|
|
||||||
if (metadata?.[3]) {
|
|
||||||
try {
|
|
||||||
dynamicForm = JSON.parse(metadata[3]);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to parse form data:', err);
|
|
||||||
dynamicForm = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isQuestion && isOptions && metadata?.[2]) {
|
|
||||||
try {
|
|
||||||
let optionsData = metadata[2];
|
|
||||||
// Use a regular expression to extract the JSON block
|
|
||||||
const jsonBlockMatch = optionsData.match(/```json([\s\S]*?)```/);
|
|
||||||
|
|
||||||
// If a JSON block is found, extract and clean it
|
|
||||||
if (jsonBlockMatch) {
|
|
||||||
optionsData = jsonBlockMatch[1].trim(); // Extract the content inside the block
|
|
||||||
} else {
|
|
||||||
// Optionally, handle the case where there is no explicit ```json block
|
|
||||||
console.warn('No JSON block found in the provided string.');
|
|
||||||
}
|
|
||||||
options = JSON.parse(optionsData);
|
|
||||||
options = options.filter(
|
|
||||||
(opt) => typeof opt.optionTitle === 'string' && typeof opt.optionDescription === 'string'
|
|
||||||
);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to parse options data:', err);
|
|
||||||
options = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move useEffect to top level
|
|
||||||
useEffect(() => {
|
|
||||||
const currentMetadataStatus = metadata?.[0];
|
|
||||||
const shouldNotify =
|
|
||||||
currentMetadataStatus &&
|
|
||||||
(currentMetadataStatus === 'QUESTION' || currentMetadataStatus === 'OPTIONS') &&
|
|
||||||
prevStatusRef.current !== currentMetadataStatus;
|
|
||||||
|
|
||||||
if (shouldNotify) {
|
|
||||||
window.electron.showNotification({
|
|
||||||
title: 'Goose has a question for you',
|
|
||||||
body: `Please check with Goose to approve the plan of action`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
prevStatusRef.current = currentMetadataStatus ?? null;
|
|
||||||
}, [metadata]);
|
|
||||||
|
|
||||||
const handleOptionClick = (index: number) => {
|
|
||||||
setSelectedOption(index);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleAccept = () => {
|
|
||||||
append('Yes - go ahead.');
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleSubmit = () => {
|
|
||||||
if (selectedOption !== null && options[selectedOption]) {
|
|
||||||
append(`Yes - continue with: ${options[selectedOption].optionTitle}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleFormSubmit = (e: React.FormEvent) => {
|
|
||||||
e.preventDefault();
|
|
||||||
if (dynamicForm) {
|
|
||||||
append(JSON.stringify(formValues));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleFormChange = (name: string, value: string) => {
|
|
||||||
setFormValues((prev) => ({
|
|
||||||
...prev,
|
|
||||||
[name]: value,
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
|
|
||||||
if (!metadata) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isForm(f: DynamicForm | null): f is DynamicForm {
|
|
||||||
return (
|
|
||||||
!!f &&
|
|
||||||
!!f.title &&
|
|
||||||
!!f.description &&
|
|
||||||
!!f.fields &&
|
|
||||||
Array.isArray(f.fields) &&
|
|
||||||
f.fields.length > 0
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="space-y-4">
|
|
||||||
{isQuestion && !isOptions && !isForm(dynamicForm) && (
|
|
||||||
<div className="flex items-center gap-4 p-4 rounded-lg bg-tool-card dark:bg-tool-card-dark border dark:border-dark-border">
|
|
||||||
<Button onClick={handleAccept} className="w-full sm:w-auto">
|
|
||||||
<Send className="h-[14px] w-[14px]" />
|
|
||||||
Take flight with this plan
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{isQuestion && isOptions && Array.isArray(options) && options.length > 0 && (
|
|
||||||
<div className="space-y-4">
|
|
||||||
{options.map((opt, index) => (
|
|
||||||
<div
|
|
||||||
key={index}
|
|
||||||
onClick={() => handleOptionClick(index)}
|
|
||||||
className={cn(
|
|
||||||
'p-4 rounded-lg border transition-colors cursor-pointer',
|
|
||||||
selectedOption === index
|
|
||||||
? 'bg-primary/10 dark:bg-dark-primary border-primary dark:border-dark-primary'
|
|
||||||
: 'bg-tool-card dark:bg-tool-card-dark hover:bg-accent dark:hover:bg-dark-accent'
|
|
||||||
)}
|
|
||||||
>
|
|
||||||
<h3 className="font-semibold text-lg mb-2 dark:text-gray-100">{opt.optionTitle}</h3>
|
|
||||||
<div className="prose prose-xs max-w-none dark:text-gray-100">
|
|
||||||
<MarkdownContent content={opt.optionDescription} />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
<Button
|
|
||||||
onClick={handleSubmit}
|
|
||||||
variant="default"
|
|
||||||
className="w-full sm:w-auto dark:bg-button-dark"
|
|
||||||
disabled={selectedOption === null}
|
|
||||||
>
|
|
||||||
<Send className="h-[14px] w-[14px]" />
|
|
||||||
Select plan
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{isForm(dynamicForm) && !isOptions && (
|
|
||||||
<form
|
|
||||||
onSubmit={handleFormSubmit}
|
|
||||||
className="space-y-4 p-4 rounded-lg bg-tool-card dark:bg-tool-card-dark border dark:border-dark-border"
|
|
||||||
>
|
|
||||||
<h2 className="text-xl font-medium mb-2 dark:text-gray-100">{dynamicForm.title}</h2>
|
|
||||||
<p className="text-sm text-gray-600 dark:text-gray-300 mb-4">{dynamicForm.description}</p>
|
|
||||||
|
|
||||||
{dynamicForm.fields.map((field) => (
|
|
||||||
<div key={field.name} className="space-y-2">
|
|
||||||
<label
|
|
||||||
htmlFor={field.name}
|
|
||||||
className="block text-sm font-medium text-gray-700 dark:text-gray-200"
|
|
||||||
>
|
|
||||||
{field.label}
|
|
||||||
{field.required && <span className="text-red-500 ml-1">*</span>}
|
|
||||||
</label>
|
|
||||||
{field.type === 'textarea' ? (
|
|
||||||
<textarea
|
|
||||||
id={field.name}
|
|
||||||
name={field.name}
|
|
||||||
placeholder={field.placeholder}
|
|
||||||
required={field.required}
|
|
||||||
value={formValues[field.name] || ''}
|
|
||||||
onChange={(e) => handleFormChange(field.name, e.target.value)}
|
|
||||||
className="w-full p-2 border rounded-md dark:bg-gray-700 dark:border-gray-600 dark:text-gray-100"
|
|
||||||
rows={4}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
id={field.name}
|
|
||||||
name={field.name}
|
|
||||||
placeholder={field.placeholder}
|
|
||||||
required={field.required}
|
|
||||||
value={formValues[field.name] || ''}
|
|
||||||
onChange={(e) => handleFormChange(field.name, e.target.value)}
|
|
||||||
className="w-full p-2 border rounded-md dark:bg-gray-700 dark:border-gray-600 dark:text-gray-100"
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
|
|
||||||
<Button type="submit" className="w-full sm:w-auto mt-4">
|
|
||||||
<Send className="h-[14px] w-[14px]" />
|
|
||||||
Submit Form
|
|
||||||
</Button>
|
|
||||||
</form>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { Card } from './ui/card';
|
|
||||||
|
|
||||||
interface Metadata {
|
|
||||||
title?: string;
|
|
||||||
description?: string;
|
|
||||||
favicon?: string;
|
|
||||||
image?: string;
|
|
||||||
url: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface LinkPreviewProps {
|
|
||||||
url: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchMetadata(url: string): Promise<Metadata> {
|
|
||||||
try {
|
|
||||||
// Fetch the HTML content using the main process
|
|
||||||
const html = await window.electron.fetchMetadata(url);
|
|
||||||
const parser = new DOMParser();
|
|
||||||
const doc = parser.parseFromString(html, 'text/html');
|
|
||||||
const baseUrl = new URL(url);
|
|
||||||
|
|
||||||
// Extract title
|
|
||||||
const title =
|
|
||||||
doc.querySelector('title')?.textContent ||
|
|
||||||
doc.querySelector('meta[property="og:title"]')?.getAttribute('content');
|
|
||||||
|
|
||||||
// Extract description
|
|
||||||
const description =
|
|
||||||
doc.querySelector('meta[name="description"]')?.getAttribute('content') ||
|
|
||||||
doc.querySelector('meta[property="og:description"]')?.getAttribute('content');
|
|
||||||
|
|
||||||
// Extract favicon
|
|
||||||
const faviconLink =
|
|
||||||
doc.querySelector('link[rel="icon"]') ||
|
|
||||||
doc.querySelector('link[rel="shortcut icon"]') ||
|
|
||||||
doc.querySelector('link[rel="apple-touch-icon"]') ||
|
|
||||||
doc.querySelector('link[rel="apple-touch-icon-precomposed"]');
|
|
||||||
|
|
||||||
let favicon = faviconLink?.getAttribute('href');
|
|
||||||
if (favicon) {
|
|
||||||
favicon = new URL(favicon, baseUrl).toString();
|
|
||||||
} else {
|
|
||||||
// Fallback to /favicon.ico
|
|
||||||
favicon = new URL('/favicon.ico', baseUrl).toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract OpenGraph image
|
|
||||||
let image = doc.querySelector('meta[property="og:image"]')?.getAttribute('content');
|
|
||||||
if (image) {
|
|
||||||
image = new URL(image, baseUrl).toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
title: title || url,
|
|
||||||
description: description || undefined,
|
|
||||||
favicon,
|
|
||||||
image: image || undefined,
|
|
||||||
url,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Error fetching metadata:', error);
|
|
||||||
return {
|
|
||||||
title: url,
|
|
||||||
description: undefined,
|
|
||||||
favicon: undefined,
|
|
||||||
image: undefined,
|
|
||||||
url,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function LinkPreview({ url }: LinkPreviewProps) {
|
|
||||||
const [metadata, setMetadata] = useState<Metadata | null>(null);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
let mounted = true;
|
|
||||||
|
|
||||||
const fetchData = async () => {
|
|
||||||
try {
|
|
||||||
const data = await fetchMetadata(url);
|
|
||||||
if (mounted) {
|
|
||||||
setMetadata(data);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (mounted) {
|
|
||||||
console.error('❌ Failed to fetch metadata:', err);
|
|
||||||
const errorMessage = err instanceof Error ? err.message : 'Failed to fetch metadata';
|
|
||||||
setError(errorMessage);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
if (mounted) {
|
|
||||||
setLoading(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
fetchData();
|
|
||||||
return () => {
|
|
||||||
mounted = false;
|
|
||||||
};
|
|
||||||
}, [url]);
|
|
||||||
|
|
||||||
if (loading) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!metadata || !metadata.title) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<a href={url} target="_blank" rel="noreferrer">
|
|
||||||
<Card className="max-w-[300px] truncate flex items-center bg-link-preview dark:bg-link-preview-dark p-3 transition-colors cursor-pointer">
|
|
||||||
{metadata.favicon && (
|
|
||||||
<img
|
|
||||||
src={metadata.favicon}
|
|
||||||
alt="Site favicon"
|
|
||||||
className="w-4 h-4 mr-2"
|
|
||||||
onError={(e) => {
|
|
||||||
e.currentTarget.style.display = 'none';
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
<div className="flex-1 min-w-0">
|
|
||||||
<h4 className="text-sm font-medium truncate">{metadata.title || url}</h4>
|
|
||||||
{metadata.description && (
|
|
||||||
<p className="text-xs text-gray-500 truncate">{metadata.description}</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
{metadata.image && (
|
|
||||||
<img
|
|
||||||
src={metadata.image}
|
|
||||||
alt="Preview"
|
|
||||||
className="w-16 h-16 object-cover rounded ml-3"
|
|
||||||
onError={(e) => {
|
|
||||||
e.currentTarget.style.display = 'none';
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</Card>
|
|
||||||
</a>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,5 @@
|
|||||||
import { useRef, useMemo, useState, useEffect, useCallback } from 'react';
|
import { useRef, useMemo, useState, useEffect, useCallback } from 'react';
|
||||||
import LinkPreview from './LinkPreview';
|
|
||||||
import ImagePreview from './ImagePreview';
|
import ImagePreview from './ImagePreview';
|
||||||
import { extractUrls } from '../utils/urlUtils';
|
|
||||||
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
|
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
|
||||||
import MarkdownContent from './MarkdownContent';
|
import MarkdownContent from './MarkdownContent';
|
||||||
import { Message, getTextContent } from '../types/message';
|
import { Message, getTextContent } from '../types/message';
|
||||||
@@ -38,9 +36,6 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
|
|||||||
// Memoize the timestamp
|
// Memoize the timestamp
|
||||||
const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
|
const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
|
||||||
|
|
||||||
// Extract URLs which explicitly contain the http:// or https:// protocol
|
|
||||||
const urls = useMemo(() => extractUrls(displayText, []), [displayText]);
|
|
||||||
|
|
||||||
// Effect to handle message content changes and ensure persistence
|
// Effect to handle message content changes and ensure persistence
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// Log content display for debugging
|
// Log content display for debugging
|
||||||
@@ -253,18 +248,6 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
|
|||||||
Edited
|
Edited
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* TODO(alexhancock): Re-enable link previews once styled well again */}
|
|
||||||
{/* TEMPORARILY DISABLED (dorien-koelemeijer): This is causing issues in properly "generating" tool calls
|
|
||||||
that contain links and prevents security scanning */}
|
|
||||||
{/* eslint-disable-next-line no-constant-binary-expression */}
|
|
||||||
{false && urls.length > 0 && (
|
|
||||||
<div className="flex flex-wrap mt-2">
|
|
||||||
{urls.map((url, index) => (
|
|
||||||
<LinkPreview key={index} url={url} />
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
// Helper to normalize URLs for comparison
|
|
||||||
function normalizeUrl(url: string): string {
|
|
||||||
try {
|
|
||||||
const parsed = new URL(url.toLowerCase());
|
|
||||||
// Remove trailing slashes and normalize protocol
|
|
||||||
return `${parsed.protocol}//${parsed.host}${parsed.pathname.replace(/\/$/, '')}${parsed.search}${parsed.hash}`;
|
|
||||||
} catch {
|
|
||||||
// If URL parsing fails, just lowercase it
|
|
||||||
return url.toLowerCase();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper to determine if a link should be included in results
|
|
||||||
function linkIsEligible(url: string): boolean {
|
|
||||||
try {
|
|
||||||
const parsed = new URL(url.toLowerCase());
|
|
||||||
const ineligibleHosts = ['localhost', '127.0.0.1'];
|
|
||||||
return !ineligibleHosts.some((host) => parsed.hostname.includes(host));
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function extractUrls(content: string, previousUrls: string[] = []): string[] {
|
|
||||||
let remainingContent = content;
|
|
||||||
const extractedUrls: string[] = [];
|
|
||||||
|
|
||||||
// First, extract markdown links
|
|
||||||
const markdownLinkRegex = /\[([^\]]+)\]\((https?:\/\/[^)]+)\)/g;
|
|
||||||
const markdownMatches = Array.from(content.matchAll(markdownLinkRegex));
|
|
||||||
|
|
||||||
// Add markdown URLs to our results and remove them from the content
|
|
||||||
markdownMatches.forEach((match) => {
|
|
||||||
extractedUrls.push(match[2]);
|
|
||||||
// Replace the entire markdown link with whitespace to preserve string indices
|
|
||||||
remainingContent = remainingContent.replace(match[0], ' '.repeat(match[0].length));
|
|
||||||
});
|
|
||||||
|
|
||||||
// Now look for standalone URLs in the remaining content
|
|
||||||
const urlRegex = /(https?:\/\/[^\s<>"']+)/g;
|
|
||||||
const urlMatches = Array.from(remainingContent.matchAll(urlRegex));
|
|
||||||
const standardUrls = urlMatches.map((match) => match[1]);
|
|
||||||
extractedUrls.push(...standardUrls);
|
|
||||||
|
|
||||||
// Remove duplicates
|
|
||||||
const uniqueCurrentUrls = [...new Set(extractedUrls)];
|
|
||||||
|
|
||||||
// Filter out ineligible URLs
|
|
||||||
const eligibleUrls = uniqueCurrentUrls.filter(linkIsEligible);
|
|
||||||
|
|
||||||
// Normalize all URLs for comparison
|
|
||||||
const normalizedPreviousUrls = previousUrls.map(normalizeUrl);
|
|
||||||
const normalizedCurrentUrls = eligibleUrls.map(normalizeUrl);
|
|
||||||
|
|
||||||
// Filter out duplicates from previous URLs
|
|
||||||
const uniqueUrls = eligibleUrls.filter((_url, index) => {
|
|
||||||
const normalized = normalizedCurrentUrls[index];
|
|
||||||
const isDuplicate = normalizedPreviousUrls.some(
|
|
||||||
(prevUrl) => normalizeUrl(prevUrl) === normalized
|
|
||||||
);
|
|
||||||
return !isDuplicate;
|
|
||||||
});
|
|
||||||
|
|
||||||
return uniqueUrls;
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user