diff --git a/.env.local.example b/.env.local.example index 9872e55dd..6166e9aa3 100644 --- a/.env.local.example +++ b/.env.local.example @@ -10,10 +10,5 @@ OPENAI_API_KEY= # Falls back to OPENAI_API_KEY if not set ANTHROPIC_API_KEY= -# Algolia search (search API key is public/read-only) -NEXT_PUBLIC_ALGOLIA_APP_ID= -NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY= -NEXT_PUBLIC_ALGOLIA_INDEX_NAME= - # Required for Contact Sales form submission (n8n → Attio pipeline) NEXT_PUBLIC_ATTIO_WEBHOOK_URL= diff --git a/.github/workflows/algolia-reindex.yml b/.github/workflows/algolia-reindex.yml deleted file mode 100644 index ace5bdbc2..000000000 --- a/.github/workflows/algolia-reindex.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Reindex Algolia after production deploy - -# Triggers a full Algolia Crawler reindex whenever a Vercel *production* -# deployment succeeds, so newly published docs pages appear in search right -# away instead of waiting for the crawler's scheduled run. -# -# Vercel's Git integration posts a GitHub deployment status of -# state=success / environment=Production once the deploy is live, which is -# what this workflow keys off of (preview and staging deploys are ignored). -# -# To avoid unnecessary crawls, the reindex is skipped when the deployed commit -# didn't touch docs content. `main` is squash-merged, so `git diff-tree HEAD` -# lists exactly the files that publish changed. -# -# Requires (repo Settings -> Secrets and variables -> Actions): -# - Variable ALGOLIA_CRAWLER_ID (fe95aa31-abbb-40ea-a31b-04a9ccd176b4) -# - Secret ALGOLIA_CRAWLER_USER_ID (from Algolia dashboard -> Crawler -> API credentials) -# - Secret ALGOLIA_CRAWLER_API_KEY (from Algolia dashboard -> Crawler -> API credentials) - -on: - deployment_status: - -# Serialize reindexes so a non-docs deploy doesn't cancel an in-flight crawl -# triggered by an earlier docs deploy. Queued runs each check their own commit; -# only runs whose commit touched docs content actually trigger a crawl. -concurrency: - group: algolia-reindex - cancel-in-progress: false - -permissions: - contents: read - -jobs: - reindex: - if: > - github.event.deployment_status.state == 'success' && - github.event.deployment_status.environment == 'Production' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.deployment.sha }} - fetch-depth: 2 - - - name: Check whether docs content changed - id: changed - run: | - files=$(git diff-tree --no-commit-id --name-only -r HEAD) - if echo "$files" | grep -qE '^app/en/|^toolkit-docs-generator/data/toolkits/|^app/_components/toolkit-docs/|^app/_lib/toolkit-|^algolia/crawler-config\.js$|^scripts/sync-crawler-config\.ts$'; then - echo "docs=true" >> "$GITHUB_OUTPUT" - echo "Docs content changed — will reindex." - else - echo "docs=false" >> "$GITHUB_OUTPUT" - echo "No docs content changed — skipping reindex." - fi - - - name: Trigger Algolia Crawler reindex - if: steps.changed.outputs.docs == 'true' - env: - CRAWLER_ID: ${{ vars.ALGOLIA_CRAWLER_ID }} - CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_USER_ID }} - CRAWLER_API_KEY: ${{ secrets.ALGOLIA_CRAWLER_API_KEY }} - run: | - echo "Triggering reindex for crawler ${CRAWLER_ID}" - curl -sS --fail-with-body -X POST \ - "https://crawler.algolia.com/api/1/crawlers/${CRAWLER_ID}/reindex" \ - -u "${CRAWLER_USER_ID}:${CRAWLER_API_KEY}" diff --git a/.github/workflows/sync-crawler-config.yml b/.github/workflows/sync-crawler-config.yml deleted file mode 100644 index 31eb7a38d..000000000 --- a/.github/workflows/sync-crawler-config.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Sync Algolia crawler config - -# Pushes algolia/crawler-config.js to the Algolia Crawler and triggers a -# reindex, so the versioned config is the source of truth and no manual -# dashboard editing is needed. -# -# Runs only when the config (or this sync script) changes on main, plus a -# manual trigger. Content-only publishes are handled by algolia-reindex.yml -# instead, so the two never double-crawl. -# -# Requires (repo Settings -> Secrets and variables -> Actions): -# - Variable ALGOLIA_CRAWLER_ID -# - Secret ALGOLIA_CRAWLER_USER_ID -# - Secret ALGOLIA_CRAWLER_API_KEY -# - Secret ALGOLIA_ADMIN_API_KEY (optional — needed to push -# initialIndexSettings to existing indices, since the crawler -# only applies them on index creation) - -on: - push: - branches: [main] - paths: - - algolia/crawler-config.js - - scripts/sync-crawler-config.ts - workflow_dispatch: - -concurrency: - group: algolia-sync-config - cancel-in-progress: true - -permissions: - contents: read - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc - - name: Sync config and trigger reindex - env: - ALGOLIA_CRAWLER_ID: ${{ vars.ALGOLIA_CRAWLER_ID }} - ALGOLIA_CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_USER_ID }} - ALGOLIA_CRAWLER_API_KEY: ${{ secrets.ALGOLIA_CRAWLER_API_KEY }} - ALGOLIA_ADMIN_API_KEY: ${{ secrets.ALGOLIA_ADMIN_API_KEY }} - # On push, skip the reindex here — algolia-reindex.yml fires after the - # Vercel production deploy so pages are live before the crawl starts. - # On workflow_dispatch there is no pending deploy, so reindex immediately. - SKIP_REINDEX: ${{ github.event_name == 'push' }} - run: pnpm sync-crawler-config diff --git a/CLAUDE.md b/CLAUDE.md index 0bdfbf7b9..775da3b84 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ pnpm vitest run tests/broken-link-check.test.ts - **`app/_lib/`** — Data-fetching utilities (toolkit catalog, slug generation, static params). - **`app/api/`** — API routes (markdown export, toolkit-data, glossary). - **`toolkit-docs-generator/`** — Generates MCP toolkit documentation from server metadata JSON files in `toolkit-docs-generator/data/toolkits/`. -- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, llms.txt generation, Algolia crawler config, i18n sync). +- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, llms.txt generation, i18n sync). - **`tests/`** — Vitest tests (broken links, internal link validation, sitemap, smoke tests). - **`lib/`** — Next.js utilities (glossary remark plugin, llmstxt plugin). - **`next.config.ts`** — Contains ~138 redirect rules. diff --git a/algolia/README.md b/algolia/README.md deleted file mode 100644 index 806f547e2..000000000 --- a/algolia/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Docs search (Algolia) - -Search on `docs.arcade.dev` is powered by an Algolia Crawler that indexes the -live site. This directory holds the crawler configuration as the **source of -truth**, so search behavior is versioned, reviewed, and changed through git. - -## Edit the config here, never in the Algolia dashboard - -`crawler-config.js` is the real configuration. Treat the Algolia dashboard -editor as generated output, not a place to make changes. - -> **Do not edit the crawler config in the Algolia UI.** Dashboard edits are not -> tracked in git, get overwritten by the next sync, and cause the live crawler -> to drift from this file. Change `crawler-config.js` and open a PR instead. - -The dashboard is still useful for **read-only** work: the URL Tester (to preview -what a config change would extract) and Monitoring (to see crawl status and -which URLs succeeded or were ignored). - -## How search stays up to date - -Two GitHub Actions workflows keep the index fresh. They never overlap. - -| When | Workflow | What happens | -| --- | --- | --- | -| Docs content is published (production deploy) | `algolia-reindex.yml` | Triggers a reindex, but only when the deploy touched `app/en/**` or `toolkit-docs-generator/data/toolkits/**` | -| `crawler-config.js` changes on `main` | `sync-crawler-config.yml` | Pushes the config to the Crawler API, then triggers a reindex | - -The crawler also runs on a weekly schedule as a backstop. - -## Change search behavior - -1. Edit `crawler-config.js` (extraction selectors, ranking, searchable - attributes, crawl scope, and so on). -2. Optional but recommended: paste the change into the dashboard URL Tester on a - representative page to confirm it extracts what you expect. -3. Open a PR. On merge, `sync-crawler-config.yml` applies it and reindexes. - -To apply the config without a merge, run the sync workflow manually from the -Actions tab (`workflow_dispatch`), or run `pnpm sync-crawler-config` locally with -the crawler credentials set as environment variables. - -## Configuration and secrets - -The crawler API key is never stored in this repo. It lives in the Algolia -dashboard and in CI secrets. The workflows need: - -| Name | Type | Source | -| --- | --- | --- | -| `ALGOLIA_CRAWLER_ID` | Variable | Crawler dashboard URL | -| `ALGOLIA_CRAWLER_USER_ID` | Secret | Crawler → API credentials | -| `ALGOLIA_CRAWLER_API_KEY` | Secret | Crawler → API credentials | -| `ALGOLIA_ADMIN_API_KEY` | Secret | Algolia dashboard → API Keys → Admin API Key | - -### Why `ALGOLIA_ADMIN_API_KEY` is needed - -The crawler's `initialIndexSettings` block is only applied when the crawler -*creates* an index. Once the production index exists, later edits to that -block (for example, tuning `removeWordsIfNoResults` to relax matching for -long agent-style queries) never reach the live index through the crawler -API alone, and the no-result rate stays elevated for those queries. - -`scripts/sync-crawler-config.ts` therefore also pushes `initialIndexSettings` -to each live index via the Search API's `PUT /1/indexes/{name}/settings` -using this key. Without the key the config still syncs and a reindex still -runs, but the index-settings step is skipped with a warning. - -The public search credentials used by the frontend widget -(`app/_components/algolia-search.tsx`) are separate `NEXT_PUBLIC_ALGOLIA_*` -environment variables. - -## Troubleshooting - -- **A crawl is blocked with `SafeReindexingError`.** The new crawl produced far - fewer records than the live index (a safety guard against a broken extractor). - Check Monitoring and the URL Tester to confirm the extraction is correct. If - the smaller result is legitimate, use "Replace production index" in the - dashboard to accept the new baseline. If it is a regression, fix the selectors - in `crawler-config.js` first. -- **A new page is missing from search.** Confirm the production deploy touched a - watched path (otherwise `algolia-reindex.yml` skips the crawl), and check that - the page appears in `sitemap.xml`. diff --git a/algolia/crawler-config.js b/algolia/crawler-config.js deleted file mode 100644 index d7bc2005e..000000000 --- a/algolia/crawler-config.js +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Algolia Crawler configuration — source of truth for the docs search index. - * - * The live crawler config is edited in the Algolia dashboard - * (Data Sources -> Crawler -> Editor). This file mirrors it so changes are - * reviewable in git instead of only living in the dashboard. - * - * To apply a change to the live crawler, edit this file — merging it to main - * runs `.github/workflows/sync-crawler-config.yml`, which pushes it via the - * Crawler API and triggers a reindex (`pnpm sync-crawler-config`). To apply it - * by hand instead, wrap the object below in `new Crawler({ ...crawlerConfig, - * apiKey })` in the dashboard editor and run a crawl. - * - * The crawler API key is intentionally NOT stored here — it lives only in the - * Algolia dashboard and in CI secrets (see .github/workflows/algolia-reindex.yml). - * - * App ID: BJB8PBSQ9T - * Index: docs_arcade_dev_bjb8pbsq9t_docsearch - * Crawler ID: fe95aa31-abbb-40ea-a31b-04a9ccd176b4 - */ -export const crawlerConfig = { - appId: "BJB8PBSQ9T", - indexPrefix: "", - rateLimit: 8, - maxUrls: null, - // Weekly backstop only — freshness is driven by reindex-on-publish - // (.github/workflows/algolia-reindex.yml). - schedule: "every 1 week", - startUrls: ["https://docs.arcade.dev"], - sitemaps: ["https://docs.arcade.dev/sitemap.xml"], - saveBackup: false, - ignoreQueryParams: ["source", "utm_*"], - // llms.txt is markdown ("- [title](url): description" lines). Crawling it - // makes the crawler mis-parse those lines into bogus URLs that 404, so keep - // it out of the crawl — it's a machine-readable export, not a page. - exclusionPatterns: ["**/cdn-cgi/**", "**/llms.txt", "**/llms-full.txt"], - actions: [ - { - indexName: "docs_arcade_dev_bjb8pbsq9t_docsearch", - pathsToMatch: ["https://docs.arcade.dev/**"], - recordExtractor: ({ $, helpers }) => { - // Strip site chrome so nav/sidebar/toc are not indexed as content. - $( - "nav, header, footer, aside, .nextra-navbar, .nextra-sidebar, .nextra-toc, .nextra-banner, .nextra-mobile-nav, .nextra-skip-nav" - ).remove(); - - return helpers.docsearch({ - recordProps: { - lvl0: { selectors: "article h1", defaultValue: "Arcade Docs" }, - lvl1: "article h2", - lvl2: "article h3", - lvl3: "article h4", - lvl4: "article h5", - lvl5: "article h6", - // Include `pre` so terms that only appear in code samples - // (e.g. `arcade_api_key`, `tool_choice`, `chat.completions`) are - // searchable — several no-result queries were code-only terms. - content: "article p, article li, article td, article pre", - }, - indexHeadings: true, - aggregateContent: true, - }); - }, - }, - ], - // The crawler applies `initialIndexSettings` only when it *creates* an - // index, so edits below never reach an existing production index by - // themselves. `scripts/sync-crawler-config.ts` also pushes these via the - // Search API when ALGOLIA_ADMIN_API_KEY is set; see algolia/README.md. - initialIndexSettings: { - docs_arcade_dev_bjb8pbsq9t_docsearch: { - distinct: true, - attributeForDistinct: "url_without_anchor", - // Long, keyword-heavy queries (often from agents/LLMs) used to return - // nothing because Algolia's default `removeWordsIfNoResults: "none"` - // requires every word to match a single record. Relax to "allOptional" - // so a query still surfaces its best partial matches instead of zero. - removeWordsIfNoResults: "allOptional", - ignorePlurals: true, - searchableAttributes: [ - "unordered(hierarchy.lvl0)", - "unordered(hierarchy.lvl1)", - "unordered(hierarchy.lvl2)", - "unordered(hierarchy.lvl3)", - "unordered(hierarchy.lvl4)", - "unordered(hierarchy.lvl5)", - "content", - ], - customRanking: ["asc(anchor)"], - attributesToRetrieve: [ - "hierarchy", - "content", - "anchor", - "url", - "url_without_anchor", - "type", - ], - attributesForFaceting: ["type", "lang"], - }, - }, -}; diff --git a/app/_components/algolia-search.tsx b/app/_components/algolia-search.tsx deleted file mode 100644 index f808c2e8d..000000000 --- a/app/_components/algolia-search.tsx +++ /dev/null @@ -1,266 +0,0 @@ -"use client"; - -import { liteClient as algoliasearch } from "algoliasearch/lite"; -import { Search } from "lucide-react"; -import { useEffect, useState } from "react"; -import { - Configure, - Highlight, - Hits, - InstantSearch, - SearchBox, - Snippet, - useInstantSearch, -} from "react-instantsearch"; - -type DocSearchHierarchy = { - lvl0: string | null; - lvl1: string | null; - lvl2: string | null; - lvl3: string | null; - lvl4: string | null; - lvl5: string | null; -}; - -type DocSearchRecord = { - objectID: string; - type?: "lvl0" | "lvl1" | "lvl2" | "lvl3" | "lvl4" | "lvl5" | "content"; - hierarchy: DocSearchHierarchy; - content: string | null; - url: string; - anchor: string | null; - // Legacy flat fields from non-docsearch indexes - title?: string; - description?: string; -}; - -const appId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID; -const searchKey = process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY; -const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME; - -const searchClient = - appId && searchKey ? algoliasearch(appId, searchKey) : null; - -function safeHref(url: string | undefined): string { - if (!url) { - return "/"; - } - if ( - url.startsWith("https://") || - (url.startsWith("/") && !url.startsWith("//")) - ) { - return url; - } - return "/"; -} - -function getHitUrl(hit: DocSearchRecord): string { - // DocSearch records include full URLs; make them relative for same-site nav - try { - const parsed = new URL(hit.url); - return safeHref(parsed.pathname + parsed.hash); - } catch { - return safeHref(hit.url); - } -} - -function SectionPath({ hit }: { hit: DocSearchRecord }) { - if (!hit.hierarchy) { - return null; - } - - const castHit = hit as unknown as Parameters[0]["hit"]; - const levelKeys = ["lvl1", "lvl2", "lvl3", "lvl4", "lvl5"] as const; - const sectionLevels = levelKeys.filter((key) => hit.hierarchy?.[key]); - - if (sectionLevels.length === 0) { - return null; - } - - return ( -
- {sectionLevels.map((key, i) => ( - - {i > 0 && ( - - )} - - - - - ))} -
- ); -} - -function HitTitle({ hit }: { hit: DocSearchRecord }) { - const castHit = hit as unknown as Parameters[0]["hit"]; - - // lvl0 is the page title in DocSearch hierarchy - if (hit.hierarchy?.lvl0) { - return ; - } - - // Fallback for legacy flat records - if (hit.title) { - return ; - } - - return Untitled; -} - -function SearchHit({ hit }: { hit: DocSearchRecord }) { - const castHit = hit as unknown as Parameters[0]["hit"]; - const isContentHit = hit.type === "content"; - - return ( - -
- -
- - {isContentHit && hit.content && ( -
- -
- )} - {!isContentHit && hit.description && ( -
- -
- )} - - ); -} - -function EmptyQuery() { - const { indexUiState } = useInstantSearch(); - if (indexUiState.query) { - return null; - } - return ( -

- Start typing to search the docs… -

- ); -} - -function NoResults() { - const { results } = useInstantSearch(); - if (!results?.query || results.nbHits > 0) { - return null; - } - return ( -

- No results for{" "} - "{results.query}" -

- ); -} - -function SearchUnavailable() { - return ( -

- Add NEXT_PUBLIC_ALGOLIA_APP_ID,{" "} - NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY, and{" "} - NEXT_PUBLIC_ALGOLIA_INDEX_NAME to your - environment to enable search. -

- ); -} - -export function AlgoliaSearch() { - const [isOpen, setIsOpen] = useState(false); - - useEffect(() => { - const handler = (e: KeyboardEvent) => { - if ((e.metaKey || e.ctrlKey) && e.key === "k") { - e.preventDefault(); - setIsOpen((prev) => !prev); - } - if (e.key === "Escape") { - setIsOpen(false); - } - }; - window.addEventListener("keydown", handler); - return () => window.removeEventListener("keydown", handler); - }, []); - - return ( - <> - - - {isOpen && ( -
-
- )} - - ); -} diff --git a/app/_components/docs-search.tsx b/app/_components/docs-search.tsx new file mode 100644 index 000000000..38c395079 --- /dev/null +++ b/app/_components/docs-search.tsx @@ -0,0 +1,263 @@ +"use client"; + +import { Search } from "lucide-react"; +import { + type KeyboardEvent as ReactKeyboardEvent, + useEffect, + useId, + useMemo, + useRef, + useState, +} from "react"; +import { createBm25Index } from "@/app/_lib/search/bm25"; +import { highlightQuery } from "@/app/_lib/search/highlight"; +import type { SearchDocument, SearchHit } from "@/app/_lib/search/types"; + +const HIT_LIMIT = 15; + +let documentsPromise: Promise | null = null; + +function fetchSearchDocuments(): Promise { + if (!documentsPromise) { + documentsPromise = fetch("/api/search-index") + .then((response) => { + if (!response.ok) { + throw new Error("Search index request failed"); + } + return response.json() as Promise<{ documents: SearchDocument[] }>; + }) + .then((payload) => payload.documents) + .catch((error: unknown) => { + documentsPromise = null; + throw error; + }); + } + return documentsPromise; +} + +function HighlightedText({ text, query }: { text: string; query: string }) { + const parts = highlightQuery(text, query); + return ( + <> + {parts.map((part, index) => + part.match ? ( + + {part.text} + + ) : ( + {part.text} + ) + )} + + ); +} + +function SearchHitLink({ + hit, + query, + active, + onSelect, +}: { + hit: SearchHit; + query: string; + active: boolean; + onSelect: () => void; +}) { + const sectionLevels = hit.heading ? [hit.heading] : []; + + return ( + +
+ +
+ {sectionLevels.length > 0 && ( +
+ {sectionLevels.map((heading) => ( + + + + ))} +
+ )} + {hit.content && ( +
+ +
+ )} + + ); +} + +export function DocsSearch() { + const [isOpen, setIsOpen] = useState(false); + const [query, setQuery] = useState(""); + const [documents, setDocuments] = useState(null); + const [loadError, setLoadError] = useState(false); + const [activeIndex, setActiveIndex] = useState(0); + const inputRef = useRef(null); + const listId = useId(); + + useEffect(() => { + fetchSearchDocuments() + .then((loaded) => { + setDocuments(loaded); + setLoadError(false); + }) + .catch(() => { + setLoadError(true); + }); + }, []); + + useEffect(() => { + const handler = (event: globalThis.KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && event.key === "k") { + event.preventDefault(); + setIsOpen((previous) => !previous); + } + if (event.key === "Escape") { + setIsOpen(false); + } + }; + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }, []); + + useEffect(() => { + if (isOpen) { + inputRef.current?.focus(); + } else { + setQuery(""); + setActiveIndex(0); + } + }, [isOpen]); + + const searchIndex = useMemo( + () => (documents ? createBm25Index(documents) : null), + [documents] + ); + + const hits = useMemo( + () => + searchIndex && query.trim() ? searchIndex.search(query, HIT_LIMIT) : [], + [searchIndex, query] + ); + + const close = () => { + setIsOpen(false); + }; + + const onQueryKeyDown = (event: ReactKeyboardEvent) => { + if (event.key === "ArrowDown" && hits.length > 0) { + event.preventDefault(); + setActiveIndex((current) => Math.min(current + 1, hits.length - 1)); + return; + } + if (event.key === "ArrowUp" && hits.length > 0) { + event.preventDefault(); + setActiveIndex((current) => Math.max(current - 1, 0)); + return; + } + if (event.key === "Enter" && hits[activeIndex]) { + event.preventDefault(); + window.location.assign(hits[activeIndex].url); + close(); + } + }; + + return ( + <> + + + {isOpen && ( +
+
+ )} + + ); +} diff --git a/app/_lib/search/bm25.ts b/app/_lib/search/bm25.ts new file mode 100644 index 000000000..3e989fe97 --- /dev/null +++ b/app/_lib/search/bm25.ts @@ -0,0 +1,224 @@ +import { tokenize, uniqueTokens } from "./tokenize"; +import type { SearchDocument, SearchHit } from "./types"; + +const K1 = 1.2; +const B = 0.75; +const IDF_SMOOTHING = 0.5; +const DEFAULT_LIMIT = 15; +const PREFIX_MIN_LENGTH = 3; +const PREFIX_BOOST = 0.4; +const HASH_SEPARATOR = "#"; + +const FIELDS = ["title", "heading", "content"] as const; +type FieldName = (typeof FIELDS)[number]; + +const FIELD_WEIGHTS: Record = { + title: 5, + heading: 3, + content: 1, +}; + +type FieldIndex = { + df: Map; + avgLength: number; +}; + +type IndexedDocument = { + document: SearchDocument; + fieldTokens: Record; + termFrequency: Record>; + fieldLength: Record; +}; + +export type Bm25Index = { + search: (query: string, limit?: number) => SearchHit[]; +}; + +function termCounts(tokens: string[]): Map { + const counts = new Map(); + for (const token of tokens) { + counts.set(token, (counts.get(token) ?? 0) + 1); + } + return counts; +} + +function inverseDocumentFrequency( + documentFrequency: number, + n: number +): number { + return Math.log( + 1 + + (n - documentFrequency + IDF_SMOOTHING) / + (documentFrequency + IDF_SMOOTHING) + ); +} + +function bm25FieldScore( + termFrequency: number, + fieldLength: number, + avgLength: number, + idf: number +): number { + if (termFrequency === 0) { + return 0; + } + const safeAvg = avgLength > 0 ? avgLength : 1; + const numerator = termFrequency * (K1 + 1); + const denominator = + termFrequency + K1 * (1 - B + B * (fieldLength / safeAvg)); + return idf * (numerator / denominator); +} + +function urlWithoutHash(url: string): string { + const hashIndex = url.indexOf(HASH_SEPARATOR); + return hashIndex === -1 ? url : url.slice(0, hashIndex); +} + +function prefixBoost(queryTokens: string[], titleTokens: string[]): number { + let boost = 0; + for (const queryToken of queryTokens) { + if (queryToken.length < PREFIX_MIN_LENGTH) { + continue; + } + const hasPrefix = titleTokens.some((titleToken) => + titleToken.startsWith(queryToken) + ); + if (hasPrefix) { + boost += PREFIX_BOOST; + } + } + return boost; +} + +function buildFieldIndexes( + indexed: IndexedDocument[], + documentCount: number +): Record { + const fields = {} as Record; + + for (const field of FIELDS) { + const df = new Map(); + let totalLength = 0; + + for (const item of indexed) { + totalLength += item.fieldLength[field]; + const seen = new Set(item.fieldTokens[field]); + for (const token of seen) { + df.set(token, (df.get(token) ?? 0) + 1); + } + } + + fields[field] = { + df, + avgLength: documentCount > 0 ? totalLength / documentCount : 0, + }; + } + + return fields; +} + +function scoreDocument( + item: IndexedDocument, + queryTokens: string[], + fieldIndexes: Record, + documentCount: number +): number { + let score = 0; + + for (const field of FIELDS) { + const fieldIndex = fieldIndexes[field]; + let fieldScore = 0; + + for (const token of queryTokens) { + const tf = item.termFrequency[field].get(token) ?? 0; + if (tf === 0) { + continue; + } + const df = fieldIndex.df.get(token) ?? 0; + fieldScore += bm25FieldScore( + tf, + item.fieldLength[field], + fieldIndex.avgLength, + inverseDocumentFrequency(df, documentCount) + ); + } + + score += fieldScore * FIELD_WEIGHTS[field]; + } + + score += prefixBoost(queryTokens, item.fieldTokens.title); + score += prefixBoost(queryTokens, item.fieldTokens.heading); + return score; +} + +function bestHitPerPage(hits: SearchHit[]): SearchHit[] { + const best = new Map(); + + for (const hit of hits) { + const pageUrl = urlWithoutHash(hit.url); + const current = best.get(pageUrl); + if (!current || hit.score > current.score) { + best.set(pageUrl, hit); + } + } + + return [...best.values()].sort((left, right) => right.score - left.score); +} + +/** + * Build an in-memory BM25 index over docs search records. + * + * Ranking is multi-field (title > heading > content). Results are collapsed + * to the best-scoring record per page so a long page cannot fill the list. + */ +export function createBm25Index(documents: SearchDocument[]): Bm25Index { + const indexed: IndexedDocument[] = documents.map((document) => { + const fieldTokens = { + title: tokenize(document.title), + heading: tokenize(document.heading ?? ""), + content: tokenize(document.content), + }; + + return { + document, + fieldTokens, + termFrequency: { + title: termCounts(fieldTokens.title), + heading: termCounts(fieldTokens.heading), + content: termCounts(fieldTokens.content), + }, + fieldLength: { + title: fieldTokens.title.length, + heading: fieldTokens.heading.length, + content: fieldTokens.content.length, + }, + }; + }); + + const documentCount = indexed.length; + const fieldIndexes = buildFieldIndexes(indexed, documentCount); + + return { + search(query: string, limit = DEFAULT_LIMIT): SearchHit[] { + const queryTokens = uniqueTokens(query); + if (queryTokens.length === 0 || documentCount === 0) { + return []; + } + + const scored: SearchHit[] = []; + for (const item of indexed) { + const score = scoreDocument( + item, + queryTokens, + fieldIndexes, + documentCount + ); + if (score > 0) { + scored.push({ ...item.document, score }); + } + } + + return bestHitPerPage(scored).slice(0, limit); + }, + }; +} diff --git a/app/_lib/search/build-index.ts b/app/_lib/search/build-index.ts new file mode 100644 index 000000000..07ac97a7c --- /dev/null +++ b/app/_lib/search/build-index.ts @@ -0,0 +1,108 @@ +import { readdir, readFile } from "node:fs/promises"; +import { dirname, join, relative } from "node:path"; +import { loadAllToolkitData } from "@/app/_lib/toolkit-data"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { documentsFromMdx } from "./mdx-documents"; +import { documentsFromToolkit } from "./toolkit-documents"; +import type { SearchDocument } from "./types"; + +const DEFAULT_PAGES_DIR = join(process.cwd(), "app", "en"); +const PAGE_MDX = "page.mdx"; + +async function collectMdxFiles(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name.startsWith("_") || entry.name.includes("[")) { + continue; + } + files.push(...(await collectMdxFiles(fullPath))); + continue; + } + if (entry.name === PAGE_MDX) { + files.push(fullPath); + } + } + + return files; +} + +function urlFromMdxPath(pagesDir: string, filePath: string): string { + const relativeDir = relative(pagesDir, dirname(filePath)).replaceAll( + "\\", + "/" + ); + if (relativeDir === "" || relativeDir === ".") { + return "/en"; + } + return `/en/${relativeDir}`; +} + +async function collectMdxDocuments( + pagesDir: string +): Promise { + const files = await collectMdxFiles(pagesDir); + const documents: SearchDocument[] = []; + + for (const filePath of files) { + const source = await readFile(filePath, "utf8"); + documents.push( + ...documentsFromMdx(urlFromMdxPath(pagesDir, filePath), source) + ); + } + + return documents; +} + +async function collectToolkitDocuments( + dataDir: string | undefined, + indexedPageUrls: Set +): Promise { + const { byNormalizedId } = await loadAllToolkitData( + dataDir ?? resolveToolkitDataDir() + ); + const documents: SearchDocument[] = []; + + for (const toolkit of byNormalizedId.values()) { + const toolkitDocuments = documentsFromToolkit(toolkit); + for (const document of toolkitDocuments) { + const isPageRecord = document.type === "page"; + if (isPageRecord && indexedPageUrls.has(document.url)) { + continue; + } + documents.push(document); + } + } + + return documents; +} + +/** + * Build the full docs search corpus from authored MDX and generated toolkit + * JSON. Dynamic-route templates (`[toolkitId]`) are skipped; toolkit pages + * come from the same JSON the site renders. + * + * When an authored MDX page already occupies a toolkit URL (a partner page), + * the MDX page record wins and the JSON page record is dropped. Tool records + * from JSON are still added. + */ +export async function buildSearchIndex(options?: { + pagesDir?: string; + dataDir?: string; +}): Promise { + const pagesDir = options?.pagesDir ?? DEFAULT_PAGES_DIR; + const mdxDocuments = await collectMdxDocuments(pagesDir); + const indexedPageUrls = new Set( + mdxDocuments + .filter((document) => document.type === "page") + .map((document) => document.url) + ); + const toolkitDocuments = await collectToolkitDocuments( + options?.dataDir, + indexedPageUrls + ); + return [...mdxDocuments, ...toolkitDocuments]; +} diff --git a/app/_lib/search/highlight.ts b/app/_lib/search/highlight.ts new file mode 100644 index 000000000..a5e13c73e --- /dev/null +++ b/app/_lib/search/highlight.ts @@ -0,0 +1,38 @@ +import { uniqueTokens } from "./tokenize"; + +const ESCAPE_REGEX = /[.*+?^${}()|[\]\\]/g; + +export type HighlightPart = { + text: string; + match: boolean; +}; + +export function escapeRegExp(value: string): string { + return value.replace(ESCAPE_REGEX, "\\$&"); +} + +/** + * Split `text` into runs that do / do not match a query token so the search + * UI can highlight hits without dangerouslySetInnerHTML. + */ +export function highlightQuery(text: string, query: string): HighlightPart[] { + if (!text) { + return []; + } + + const tokens = uniqueTokens(query); + if (tokens.length === 0) { + return [{ text, match: false }]; + } + + const pattern = new RegExp(`(${tokens.map(escapeRegExp).join("|")})`, "gi"); + const parts = text.split(pattern); + const tokenSet = new Set(tokens); + + return parts + .filter((part) => part.length > 0) + .map((part) => ({ + text: part, + match: tokenSet.has(part.toLowerCase()), + })); +} diff --git a/app/_lib/search/mdx-documents.ts b/app/_lib/search/mdx-documents.ts new file mode 100644 index 000000000..052feba34 --- /dev/null +++ b/app/_lib/search/mdx-documents.ts @@ -0,0 +1,212 @@ +import { parse as parseYaml } from "yaml"; +import type { SearchDocument } from "./types"; + +const FRONTMATTER_REGEX = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/; +const HEADING_REGEX = /^(#{1,6})\s+(.+)$/; +const FENCE_REGEX = /^```/; +const IMPORT_EXPORT_START_REGEX = /^(?:import|export)\s/; +const FENCED_BLOCK_REGEX = /```[\s\S]*?```/g; +const FENCE_OPEN_REGEX = /^```\w*\n?/; +const FENCE_CLOSE_REGEX = /```$/; +const INLINE_CODE_REGEX = /`([^`]+)`/g; +const IMAGE_REGEX = /!\[([^\]]*)\]\([^)]+\)/g; +const LINK_REGEX = /\[([^\]]+)\]\([^)]+\)/g; +const JSX_TAG_REGEX = /<\/?[A-Za-z][^>]*>/g; +const MARKDOWN_NOISE_REGEX = /[*_~#>]+/g; +const WHITESPACE_REGEX = /\s+/g; +const NON_SLUG_REGEX = /[^a-z0-9\s-]/g; +const MULTI_HYPHEN_REGEX = /-+/g; + +const MAX_CONTENT_CHARS = 2000; +const MIN_HEADING_LEVEL = 2; + +type Frontmatter = { + title?: string; + description?: string; +}; + +type Section = { + level: number; + heading: string; + content: string; +}; + +function truncateContent(text: string): string { + if (text.length <= MAX_CONTENT_CHARS) { + return text; + } + const sliced = text.slice(0, MAX_CONTENT_CHARS); + const lastSpace = sliced.lastIndexOf(" "); + const cut = lastSpace > 0 ? sliced.slice(0, lastSpace) : sliced; + return `${cut}…`; +} + +export function slugifyHeading(heading: string): string { + return heading + .toLowerCase() + .trim() + .replace(NON_SLUG_REGEX, "") + .replace(WHITESPACE_REGEX, "-") + .replace(MULTI_HYPHEN_REGEX, "-"); +} + +export function stripMarkdown(text: string): string { + return text + .replace(FENCED_BLOCK_REGEX, (block) => + block.replace(FENCE_OPEN_REGEX, "").replace(FENCE_CLOSE_REGEX, "") + ) + .replace(INLINE_CODE_REGEX, "$1") + .replace(IMAGE_REGEX, "$1") + .replace(LINK_REGEX, "$1") + .replace(JSX_TAG_REGEX, " ") + .replace(MARKDOWN_NOISE_REGEX, " ") + .replace(WHITESPACE_REGEX, " ") + .trim(); +} + +function parseFrontmatter(source: string): { + data: Frontmatter; + body: string; +} { + const match = source.match(FRONTMATTER_REGEX); + if (!match) { + return { data: {}, body: source }; + } + + try { + const parsed: unknown = parseYaml(match[1]); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const record = parsed as Record; + return { + data: { + title: typeof record.title === "string" ? record.title : undefined, + description: + typeof record.description === "string" + ? record.description + : undefined, + }, + body: source.slice(match[0].length), + }; + } + } catch { + // Invalid YAML — index the raw body instead of failing the whole index. + } + + return { data: {}, body: source }; +} + +function stripImportsAndExports(source: string): string { + const lines = source.split("\n"); + const kept: string[] = []; + let skipping = false; + + for (const line of lines) { + const trimmed = line.trim(); + if (skipping) { + if (trimmed.endsWith(";")) { + skipping = false; + } + continue; + } + if (IMPORT_EXPORT_START_REGEX.test(trimmed)) { + skipping = !trimmed.endsWith(";"); + continue; + } + kept.push(line); + } + + return kept.join("\n"); +} + +function splitSections(body: string): Section[] { + const sections: Section[] = [{ level: 0, heading: "", content: "" }]; + let inFence = false; + + for (const line of body.split("\n")) { + if (FENCE_REGEX.test(line.trim())) { + inFence = !inFence; + const current = sections.at(-1); + if (current) { + current.content += `${line}\n`; + } + continue; + } + + const headingMatch = inFence ? null : line.match(HEADING_REGEX); + if (headingMatch) { + sections.push({ + level: headingMatch[1].length, + heading: headingMatch[2].trim(), + content: "", + }); + continue; + } + + const current = sections.at(-1); + if (current) { + current.content += `${line}\n`; + } + } + + return sections; +} + +function joinNonEmpty(parts: Array): string { + return parts + .filter((part): part is string => Boolean(part?.trim())) + .join(" "); +} + +/** + * Turn one authored MDX page into search records: a page-level document plus + * one record per `##` (or deeper) section so BM25 can deep-link to headings. + */ +export function documentsFromMdx( + url: string, + source: string +): SearchDocument[] { + const { data, body } = parseFrontmatter(source); + const sections = splitSections(stripImportsAndExports(body)); + const h1 = sections.find((section) => section.level === 1); + const pageTitle = data.title?.trim() || h1?.heading || "Untitled"; + const preamble = stripMarkdown( + joinNonEmpty( + sections + .filter((section) => section.level <= 1) + .map((section) => section.content) + ) + ); + + const documents: SearchDocument[] = [ + { + id: url, + url, + title: pageTitle, + heading: null, + content: truncateContent(joinNonEmpty([data.description, preamble])), + type: "page", + }, + ]; + + for (const section of sections) { + if (section.level < MIN_HEADING_LEVEL) { + continue; + } + const heading = stripMarkdown(section.heading); + const content = truncateContent(stripMarkdown(section.content)); + if (!(heading || content)) { + continue; + } + const sectionUrl = `${url}#${slugifyHeading(heading || section.heading)}`; + documents.push({ + id: sectionUrl, + url: sectionUrl, + title: pageTitle, + heading: heading || null, + content, + type: content ? "content" : "heading", + }); + } + + return documents; +} diff --git a/app/_lib/search/tokenize.ts b/app/_lib/search/tokenize.ts new file mode 100644 index 000000000..d84735c89 --- /dev/null +++ b/app/_lib/search/tokenize.ts @@ -0,0 +1,29 @@ +const MIN_TOKEN_LENGTH = 2; +const CAMEL_BOUNDARY_REGEX = /([a-z0-9])([A-Z])/g; +const ACRONYM_BOUNDARY_REGEX = /([A-Z]+)([A-Z][a-z])/g; +const NON_ALNUM_REGEX = /[^a-z0-9]+/g; + +/** + * Split text into lowercase search tokens. + * + * CamelCase, snake_case, and dotted names become separate tokens so queries + * like "create issue" match `Github.CreateIssue`. + */ +export function tokenize(text: string): string[] { + if (!text) { + return []; + } + + const withBoundaries = text + .replace(ACRONYM_BOUNDARY_REGEX, "$1 $2") + .replace(CAMEL_BOUNDARY_REGEX, "$1 $2"); + + return withBoundaries + .toLowerCase() + .split(NON_ALNUM_REGEX) + .filter((token) => token.length >= MIN_TOKEN_LENGTH); +} + +export function uniqueTokens(text: string): string[] { + return [...new Set(tokenize(text))]; +} diff --git a/app/_lib/search/toolkit-documents.ts b/app/_lib/search/toolkit-documents.ts new file mode 100644 index 000000000..28337911d --- /dev/null +++ b/app/_lib/search/toolkit-documents.ts @@ -0,0 +1,91 @@ +import type { ToolkitData } from "@/app/_components/toolkit-docs/types"; +import { getToolkitCanonicalPath } from "@/app/_lib/toolkit-static-params"; +import { stripMarkdown } from "./mdx-documents"; +import type { SearchDocument } from "./types"; + +const MAX_CONTENT_CHARS = 2000; +const DOT_REGEX = /\./g; +const WHITESPACE_REGEX = /\s+/g; + +function truncateContent(text: string): string { + if (text.length <= MAX_CONTENT_CHARS) { + return text; + } + const sliced = text.slice(0, MAX_CONTENT_CHARS); + const lastSpace = sliced.lastIndexOf(" "); + const cut = lastSpace > 0 ? sliced.slice(0, lastSpace) : sliced; + return `${cut}…`; +} + +function joinNonEmpty(parts: Array): string { + return parts + .filter((part): part is string => Boolean(part?.trim())) + .join(" "); +} + +/** + * Match the toolkit page's tool anchors (`toToolAnchorId`) without importing + * the client table component into the server index builder. + */ +function toToolAnchorId(value: string): string { + return value + .toLowerCase() + .replace(WHITESPACE_REGEX, "-") + .replace(DOT_REGEX, ""); +} + +/** + * Index a generated toolkit page and each of its tools. + * + * Hidden toolkits are omitted. A missing category is skipped rather than + * failing the index — those toolkits are not routed on the site either. + */ +export function documentsFromToolkit(toolkit: ToolkitData): SearchDocument[] { + if (toolkit.metadata.isHidden) { + return []; + } + + let path: string; + try { + path = getToolkitCanonicalPath({ + id: toolkit.id, + category: toolkit.metadata.category, + docsLink: toolkit.metadata.docsLink, + }); + } catch { + return []; + } + + const title = toolkit.label || toolkit.id; + const pageContent = truncateContent( + stripMarkdown(joinNonEmpty([toolkit.summary, toolkit.description])) + ); + + const documents: SearchDocument[] = [ + { + id: path, + url: path, + title, + heading: null, + content: pageContent, + type: "page", + }, + ]; + + for (const tool of toolkit.tools) { + const url = `${path}#${toToolAnchorId(tool.qualifiedName)}`; + const parameterNames = tool.parameters.map((parameter) => parameter.name); + documents.push({ + id: url, + url, + title, + heading: tool.qualifiedName, + content: truncateContent( + joinNonEmpty([tool.description, parameterNames.join(" ")]) + ), + type: "tool", + }); + } + + return documents; +} diff --git a/app/_lib/search/types.ts b/app/_lib/search/types.ts new file mode 100644 index 000000000..70bd7e344 --- /dev/null +++ b/app/_lib/search/types.ts @@ -0,0 +1,18 @@ +export type SearchDocumentType = "page" | "heading" | "content" | "tool"; + +export type SearchDocument = { + id: string; + url: string; + title: string; + heading: string | null; + content: string; + type: SearchDocumentType; +}; + +export type SearchHit = SearchDocument & { + score: number; +}; + +export type SearchIndexPayload = { + documents: SearchDocument[]; +}; diff --git a/app/api/search-index/route.ts b/app/api/search-index/route.ts new file mode 100644 index 000000000..c8d338937 --- /dev/null +++ b/app/api/search-index/route.ts @@ -0,0 +1,16 @@ +import { NextResponse } from "next/server"; +import { buildSearchIndex } from "@/app/_lib/search/build-index"; + +// Next.js requires a string literal here — a ternary is rejected at build +// (`invalid-page-config`). `next dev` still re-runs the handler on each +// request, so local edits to MDX or toolkit JSON show up without a restart. +export const dynamic = "force-static"; + +const CACHE_HEADERS = { + "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400", +}; + +export async function GET() { + const documents = await buildSearchIndex(); + return NextResponse.json({ documents }, { headers: CACHE_HEADERS }); +} diff --git a/app/en/references/changelog/page.mdx b/app/en/references/changelog/page.mdx index 2d6386ec9..73892c046 100644 --- a/app/en/references/changelog/page.mdx +++ b/app/en/references/changelog/page.mdx @@ -9,6 +9,12 @@ import { Callout } from "nextra/components"; _Here's what's new at Arcade.dev!_ +## 2026-09-11 + +**Docs** + +- `[feature - 🚀]` Replace Algolia docs search with in-memory BM25 so preview deployments search the current branch. + ## 2026-07-26 **Misc** diff --git a/app/globals.css b/app/globals.css index 1d193b7cf..d8903c5d4 100644 --- a/app/globals.css +++ b/app/globals.css @@ -166,7 +166,7 @@ nav > a[aria-label="Home page"] { margin-inline-end: 3.5rem !important; } -nav > div:has(.algolia-search-button) { +nav > div:has(.docs-search-button) { order: -1; margin-inline-start: 1rem; margin-inline-end: auto; @@ -178,9 +178,8 @@ nav > div:has(.algolia-search-button) { } } -/* Algolia search hit highlight — brand red */ -.ais-Highlight-highlighted, -.ais-Snippet-highlighted { +/* In-memory search hit highlight — brand accent */ +.docs-search-highlight { background: color-mix(in oklch, var(--primary) 15%, transparent); color: var(--primary); border-radius: 2px; @@ -188,8 +187,7 @@ nav > div:has(.algolia-search-button) { font-weight: 600; } -.dark .ais-Highlight-highlighted, -.dark .ais-Snippet-highlighted { +.dark .docs-search-highlight { background: color-mix(in oklch, var(--primary) 20%, transparent); } diff --git a/app/layout.tsx b/app/layout.tsx index 12837dabe..ad335a63c 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,8 +1,8 @@ import { getDictionary } from "@/_dictionaries/get-dictionary"; -import { AlgoliaSearch } from "@/app/_components/algolia-search"; import { SignupLink } from "@/app/_components/analytics"; import CustomLayout from "@/app/_components/custom-layout"; import { getDashboardUrl } from "@/app/_components/dashboard-link"; +import { DocsSearch } from "@/app/_components/docs-search"; import { Footer } from "@/app/_components/footer"; import { Logo } from "@/app/_components/logo"; import NavBarButton from "@/app/_components/nav-bar-button"; @@ -194,7 +194,7 @@ export default async function RootLayout({ } nextThemes={{ defaultTheme: "dark" }} pageMap={pageMap} - search={} + search={} sidebar={{ defaultMenuCollapseLevel: 2, autoCollapse: true, diff --git a/package.json b/package.json index 6298b1db8..05b7227f5 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "check-redirects": "pnpm exec tsx scripts/check-redirects.ts", "update-links": "pnpm exec tsx scripts/update-internal-links.ts", "check-meta": "pnpm exec tsx scripts/check-meta-keys.ts", - "sync-crawler-config": "pnpm exec tsx scripts/sync-crawler-config.ts", "metadata-report": "pnpm exec tsx toolkit-docs-generator/scripts/report-tool-metadata.ts" }, "repository": { @@ -47,7 +46,6 @@ "@next/third-parties": "16.3.2", "@ory/client": "1.22.37", "@uidotdev/usehooks": "2.4.1", - "algoliasearch": "5.53.0", "lucide-react": "0.577.0", "motion": "12.40.0", "next": "16.3.2", @@ -58,7 +56,6 @@ "react": "19.2.7", "react-dom": "19.2.7", "react-hook-form": "7.77.0", - "react-instantsearch": "7.34.0", "react-markdown": "10.1.0", "react-syntax-highlighter": "16.1.1", "remark-gfm": "4.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 73b75cb60..fd8caa373 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,9 +42,6 @@ importers: '@uidotdev/usehooks': specifier: 2.4.1 version: 2.4.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - algoliasearch: - specifier: 5.53.0 - version: 5.53.0 lucide-react: specifier: 0.577.0 version: 0.577.0(react@19.2.7) @@ -75,9 +72,6 @@ importers: react-hook-form: specifier: 7.77.0 version: 7.77.0(react@19.2.7) - react-instantsearch: - specifier: 7.34.0 - version: 7.34.0(algoliasearch@5.53.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) react-markdown: specifier: 10.1.0 version: 10.1.0(@types/react@19.2.16)(react@19.2.7) @@ -221,65 +215,6 @@ packages: peerDependencies: react: ^18 || ~19.0.1 || ~19.1.2 || ^19.2.1 - '@algolia/abtesting@1.19.0': - resolution: {integrity: sha512-Lhnez3hhXHk25lfxLAMxvkP4fmN3+1RgADhD2ssMDBYuAsDVReeyP+3SGRx+ntq8ijMrLqUyfvO72TB6jsTteQ==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-abtesting@5.53.0': - resolution: {integrity: sha512-0ZjA5Hcmaoz5Lj6OG0zhfIyeqzJZnLW2CRJA1W17UwMFGRtZAJ9yJKRvPEDA6gkpsIoQxORTSW6sWFiuYncPNQ==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-analytics@5.53.0': - resolution: {integrity: sha512-kWNodP75iiEaOtemC9F/hlxNBG5E2QUjN1BusnE6m2b4l7Qh/BUO3fGCVsmKJI65VO4VKGGmT43ICvHtTcJ2JQ==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-common@5.53.0': - resolution: {integrity: sha512-YPN45TXD9Wrse185t/Ta7nktZsqpv97oOjCzp2sblHnCL6rBc9TDeJAg1IGl2UpdwnSD05Zu/5wLB4watOUMyg==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-insights@5.53.0': - resolution: {integrity: sha512-qAcYTDJE6m924FDDUQvdD6vh7DYaqOeSpFS74IP37/JRV0v4cGBauyxTF2WzDnokUylQDbqreoFIJZfg0Fitmw==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-personalization@5.53.0': - resolution: {integrity: sha512-fQaY+DkSJOpuUVUe8MQTwrdiKAqkJGhpDarB08duBn/sUv7Bkib6MDRQauCcWTWTe4HIW+EbwQP9R4kci1V/Yw==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-query-suggestions@5.53.0': - resolution: {integrity: sha512-o72tsiEZGfeS/dxL9IADfzcZWGEwKDEe5CvtrBuT//3JR+SHuTtHRI2ZTf7D7bcKagcbojvO8hnkHdfoakSlYg==} - engines: {node: '>= 14.0.0'} - - '@algolia/client-search@5.53.0': - resolution: {integrity: sha512-Ds16IyPm/dNJPCU8OzApo2gwGrgWT5BYHhE3NFwZbpCveqyvPDB9sZDDkJ5DsdOGT2aC+R3i0/M1OVXF2qdgPg==} - engines: {node: '>= 14.0.0'} - - '@algolia/events@4.0.1': - resolution: {integrity: sha512-FQzvOCgoFXAbf5Y6mYozw2aj5KCJoA3m4heImceldzPSMbdyS4atVjJzXKMsfX3wnZTFYwkkt8/z8UesLHlSBQ==} - - '@algolia/ingestion@1.53.0': - resolution: {integrity: sha512-oNbT6z4NwD8Pou9VPINGlN/tlG1afESh2EbxqnP6rwl95xKVD/Zlciis1PpNeO/9U/rrajc1+7DcfKi03tX1KQ==} - engines: {node: '>= 14.0.0'} - - '@algolia/monitoring@1.53.0': - resolution: {integrity: sha512-G+KZb/yd+qAOFn/cEvTGeLxQm8aP3a0od50l3z/ylccY+/o4YG3TNcjU1tFQHW4mXC137GPyR7W70R0kRQDLnA==} - engines: {node: '>= 14.0.0'} - - '@algolia/recommend@5.53.0': - resolution: {integrity: sha512-6aVfYd55Un6IUgPLbo84WfgFZlS3L0vA1ttzXL5vahHewUJ8jYgd89TzlWRTeej7w70mb9RWsVlFYGmJ/diQww==} - engines: {node: '>= 14.0.0'} - - '@algolia/requester-browser-xhr@5.53.0': - resolution: {integrity: sha512-ke27DqgzCOlt+RbeEdCxtXxMQOnAOi8ujr2wid0DmDKzR95Kw/f9sBsuhBxtjevCqJRJszfRTLY0B1pbO6IhkA==} - engines: {node: '>= 14.0.0'} - - '@algolia/requester-fetch@5.53.0': - resolution: {integrity: sha512-GngiOqt2Gq4oLno6yXQVj9om+qSO9SWAoduoTOEg79dKZ62brB8OOIvSJG/vDNoanYi6a7Al9uDZwXvi+bcVTg==} - engines: {node: '>= 14.0.0'} - - '@algolia/requester-node-http@5.53.0': - resolution: {integrity: sha512-6mF9LZMUk0QqWvrnxkxBqhswwz6Xfiwy6/gmTzL5HrlhdVG3ITAqGV2k3XmVThP1h0Ulc3VQwiNCD7/Nr4JNlQ==} - engines: {node: '>= 14.0.0'} - '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -1948,9 +1883,6 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - '@types/dom-speech-recognition@0.0.1': - resolution: {integrity: sha512-udCxb8DvjcDKfk1WTBzDsxFbLgYxmQGKrE/ricoMqHRNjSlSUCcamVTA5lIQqzY10mY5qCY0QDwBfFEwhfoDPw==} - '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -1963,15 +1895,9 @@ packages: '@types/geojson@7946.0.16': resolution: {integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==} - '@types/google.maps@3.58.1': - resolution: {integrity: sha512-X9QTSvGJ0nCfMzYOnaVs/k6/4L+7F5uCS+4iUmkLEls6J9S/Phv+m/i3mDeyc49ZBgwab3EFO1HEoBY7k98EGQ==} - '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} - '@types/hogan.js@3.0.5': - resolution: {integrity: sha512-/uRaY3HGPWyLqOyhgvW9Aa43BNnLZrNeQxl2p8wqId4UHMfPKolSB+U7BlZyO1ng7MkLnyEAItsBzCG0SDhqrA==} - '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} @@ -1993,9 +1919,6 @@ packages: '@types/prismjs@1.26.5': resolution: {integrity: sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==} - '@types/qs@6.14.0': - resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==} - '@types/ramda@0.30.2': resolution: {integrity: sha512-PyzHvjCalm2BRYjAU6nIB3TprYwMNOUY/7P/N8bSzp9W/yM2YrtGtAnnVtaCNSeOZ8DzKyFDvaqQs7LnWwwmBA==} @@ -2081,9 +2004,6 @@ packages: resolution: {integrity: sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A==} engines: {node: '>=14.6'} - abbrev@1.1.1: - resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} - acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -2104,15 +2024,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - algoliasearch-helper@3.29.1: - resolution: {integrity: sha512-6ck2YFudF2Pje7szQoPBiRFTGfd+1I+0I/WfLPGn0bj1kvrFoOQmNyedNiDxTk3/r4IfSLDYk+RA4G7u8H6+yA==} - peerDependencies: - algoliasearch: '>= 3.1 < 6' - - algoliasearch@5.53.0: - resolution: {integrity: sha512-OGW1q6b91CRSSeiOnM8LxuR5NYJ2esvw66jUZ4IIvdv+ItNkx3pwLuyR+jaCdbGee4ov5WgUnyPryyh11xvByQ==} - engines: {node: '>= 14.0.0'} - ansi-escapes@7.2.0: resolution: {integrity: sha512-g6LhBsl+GBPRWGWsBtutpzBYuIIdBkLEvad5C/va/74Db018+5TZiyA26cZJAr3Rft5lprVqOIPxf5Vid6tqAw==} engines: {node: '>=18'} @@ -2862,13 +2773,6 @@ packages: highlightjs-vue@1.0.0: resolution: {integrity: sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==} - hogan.js@3.0.2: - resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==} - hasBin: true - - htm@3.1.1: - resolution: {integrity: sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==} - html-url-attributes@3.0.1: resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==} @@ -2910,14 +2814,6 @@ packages: inline-style-parser@0.2.7: resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} - instantsearch-ui-components@0.29.0: - resolution: {integrity: sha512-/7bY63BM+nRs38vL/1hYtjheDlf3FEK2edsnc3UEA3JJZZZk0Bq1Ds0o8WcSVymT/BSZz8DNb87eLDTz6gDVcw==} - - instantsearch.js@4.100.0: - resolution: {integrity: sha512-8fomIg5pwSTDDQQL/yUNWdc0lvvw+1xhjgNlWJwgmRsonZ3GlLT00bvznfrTHq0IRw9/Z894uMQEpH0t/l742A==} - peerDependencies: - algoliasearch: '>= 3.1 < 6' - internmap@1.0.1: resolution: {integrity: sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==} @@ -3173,15 +3069,6 @@ packages: markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} - markdown-to-jsx@7.7.17: - resolution: {integrity: sha512-7mG/1feQ0TX5I7YyMZVDgCC/y2I3CiEhIRQIhyov9nGBP5eoVrOXXHuL5ZP8GRfxVZKRiXWJgwXkb9It+nQZfQ==} - engines: {node: '>= 10'} - peerDependencies: - react: '>= 0.14.0' - peerDependenciesMeta: - react: - optional: true - marked@16.4.2: resolution: {integrity: sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==} engines: {node: '>= 20'} @@ -3412,10 +3299,6 @@ packages: mj-context-menu@0.6.1: resolution: {integrity: sha512-7NO5s6n10TIV96d4g2uDpG7ZDpIhMh0QNfGdJw/W47JswFcosz457wqz/b5sAKvl12sxINGFCn80NZHKwxQEXA==} - mkdirp@0.3.0: - resolution: {integrity: sha512-OHsdUcVAQ6pOtg5JYWpCBo9W/GySVuwvP9hueRMW7UqshC0tbfzLv8wjySTPm3tfUZ/21CE9E1pJagOA91Pxew==} - deprecated: Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.) - mlly@1.8.0: resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} @@ -3526,10 +3409,6 @@ packages: resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} hasBin: true - nopt@1.0.10: - resolution: {integrity: sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==} - hasBin: true - npm-run-path@5.3.0: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3547,10 +3426,6 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} - object-inspect@1.13.4: - resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} - engines: {node: '>= 0.4'} - obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} @@ -3696,10 +3571,6 @@ packages: resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} engines: {node: '>=10'} - qs@6.16.0: - resolution: {integrity: sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==} - engines: {node: '>=0.6'} - query-selector-shadow-dom@1.0.1: resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} @@ -3778,19 +3649,6 @@ packages: peerDependencies: react: ^16.8.4 || ^17.0.0 || ^18.0.0 - react-instantsearch-core@7.34.0: - resolution: {integrity: sha512-TnQdiIR44hHn/3Ef187Bu/bnTKxwk/Wi6zyDgft0pcXfjjGu4/7Y8X1FA27akXU20XzK3R+E2uizjOB50EBdRQ==} - peerDependencies: - algoliasearch: '>= 3.1 < 6' - react: '>= 16.8.0 < 20' - - react-instantsearch@7.34.0: - resolution: {integrity: sha512-qQ2tKQUyxluyb9ACjXXKiLzAnlcyELPuLMmkF/gCQXe3Sr7SKKhoabW+oEQHTNsBQBeOKRnxKGiLbECZ+VyKzw==} - peerDependencies: - algoliasearch: '>= 3.1 < 6' - react: '>= 16.8.0 < 20' - react-dom: '>= 16.8.0 < 20' - react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} @@ -4062,9 +3920,6 @@ packages: scroll-into-view-if-needed@3.1.0: resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} - search-insights@2.17.3: - resolution: {integrity: sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==} - semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -4124,22 +3979,6 @@ packages: resolution: {integrity: sha512-KRT/hufMSxXKEDSQujfVE0Faa/kZ51ihUcZQAcmP04t00DvPj7Ox5anHke1sJYUtzSuiT/Y5uyzg/W7bBEGhCg==} hasBin: true - side-channel-list@1.0.1: - resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} - engines: {node: '>= 0.4'} - - side-channel-map@1.0.1: - resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} - engines: {node: '>= 0.4'} - - side-channel-weakmap@1.0.2: - resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} - engines: {node: '>= 0.4'} - - side-channel@1.1.1: - resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} - engines: {node: '>= 0.4'} - siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -4722,92 +4561,6 @@ snapshots: transitivePeerDependencies: - zod - '@algolia/abtesting@1.19.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-abtesting@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-analytics@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-common@5.53.0': {} - - '@algolia/client-insights@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-personalization@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-query-suggestions@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/client-search@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/events@4.0.1': {} - - '@algolia/ingestion@1.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/monitoring@1.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/recommend@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - - '@algolia/requester-browser-xhr@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - - '@algolia/requester-fetch@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - - '@algolia/requester-node-http@5.53.0': - dependencies: - '@algolia/client-common': 5.53.0 - '@alloc/quick-lru@5.2.0': {} '@antfu/install-pkg@1.1.0': @@ -6659,8 +6412,6 @@ snapshots: '@types/deep-eql@4.0.2': {} - '@types/dom-speech-recognition@0.0.1': {} - '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.8 @@ -6671,14 +6422,10 @@ snapshots: '@types/geojson@7946.0.16': {} - '@types/google.maps@3.58.1': {} - '@types/hast@3.0.5': dependencies: '@types/unist': 3.0.3 - '@types/hogan.js@3.0.5': {} - '@types/katex@0.16.7': {} '@types/mdast@4.0.4': @@ -6699,8 +6446,6 @@ snapshots: '@types/prismjs@1.26.5': {} - '@types/qs@6.14.0': {} - '@types/ramda@0.30.2': dependencies: types-ramda: 0.30.1 @@ -6792,8 +6537,6 @@ snapshots: '@xmldom/xmldom@0.9.12': {} - abbrev@1.1.1: {} - acorn-jsx@5.3.2(acorn@8.15.0): dependencies: acorn: 8.15.0 @@ -6813,28 +6556,6 @@ snapshots: '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) zod: 4.3.6 - algoliasearch-helper@3.29.1(algoliasearch@5.53.0): - dependencies: - '@algolia/events': 4.0.1 - algoliasearch: 5.53.0 - - algoliasearch@5.53.0: - dependencies: - '@algolia/abtesting': 1.19.0 - '@algolia/client-abtesting': 5.53.0 - '@algolia/client-analytics': 5.53.0 - '@algolia/client-common': 5.53.0 - '@algolia/client-insights': 5.53.0 - '@algolia/client-personalization': 5.53.0 - '@algolia/client-query-suggestions': 5.53.0 - '@algolia/client-search': 5.53.0 - '@algolia/ingestion': 1.53.0 - '@algolia/monitoring': 1.53.0 - '@algolia/recommend': 5.53.0 - '@algolia/requester-browser-xhr': 5.53.0 - '@algolia/requester-fetch': 5.53.0 - '@algolia/requester-node-http': 5.53.0 - ansi-escapes@7.2.0: dependencies: environment: 1.1.0 @@ -7688,13 +7409,6 @@ snapshots: highlightjs-vue@1.0.0: {} - hogan.js@3.0.2: - dependencies: - mkdirp: 0.3.0 - nopt: 1.0.10 - - htm@3.1.1: {} - html-url-attributes@3.0.1: {} html-void-elements@3.0.0: {} @@ -7726,31 +7440,6 @@ snapshots: inline-style-parser@0.2.7: {} - instantsearch-ui-components@0.29.0(react@19.2.7): - dependencies: - '@swc/helpers': 0.5.18 - markdown-to-jsx: 7.7.17(react@19.2.7) - transitivePeerDependencies: - - react - - instantsearch.js@4.100.0(algoliasearch@5.53.0): - dependencies: - '@algolia/events': 4.0.1 - '@swc/helpers': 0.5.18 - '@types/dom-speech-recognition': 0.0.1 - '@types/google.maps': 3.58.1 - '@types/hogan.js': 3.0.5 - '@types/qs': 6.14.0 - algoliasearch: 5.53.0 - algoliasearch-helper: 3.29.1(algoliasearch@5.53.0) - hogan.js: 3.0.2 - htm: 3.1.1 - instantsearch-ui-components: 0.29.0(react@19.2.7) - preact: 10.29.1 - qs: 6.16.0 - react: 19.2.7 - search-insights: 2.17.3 - internmap@1.0.1: {} internmap@2.0.3: {} @@ -7960,10 +7649,6 @@ snapshots: markdown-table@3.0.4: {} - markdown-to-jsx@7.7.17(react@19.2.7): - optionalDependencies: - react: 19.2.7 - marked@16.4.2: {} marked@17.0.3: {} @@ -8505,8 +8190,6 @@ snapshots: mj-context-menu@0.6.1: {} - mkdirp@0.3.0: {} - mlly@1.8.0: dependencies: acorn: 8.15.0 @@ -8661,10 +8344,6 @@ snapshots: node-gyp-build@4.8.4: optional: true - nopt@1.0.10: - dependencies: - abbrev: 1.1.1 - npm-run-path@5.3.0: dependencies: path-key: 4.0.0 @@ -8679,8 +8358,6 @@ snapshots: object-assign@4.1.1: {} - object-inspect@1.13.4: {} - obug@2.1.1: {} onetime@6.0.0: @@ -8843,11 +8520,6 @@ snapshots: proxy-from-env@2.1.0: {} - qs@6.16.0: - dependencies: - es-define-property: 1.0.1 - side-channel: 1.1.1 - query-selector-shadow-dom@1.0.1: {} querystringify@2.2.0: {} @@ -8917,25 +8589,6 @@ snapshots: dependencies: react: 19.2.7 - react-instantsearch-core@7.34.0(algoliasearch@5.53.0)(react@19.2.7): - dependencies: - '@swc/helpers': 0.5.18 - algoliasearch: 5.53.0 - algoliasearch-helper: 3.29.1(algoliasearch@5.53.0) - instantsearch.js: 4.100.0(algoliasearch@5.53.0) - react: 19.2.7 - use-sync-external-store: 1.6.0(react@19.2.7) - - react-instantsearch@7.34.0(algoliasearch@5.53.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7): - dependencies: - '@swc/helpers': 0.5.18 - algoliasearch: 5.53.0 - instantsearch-ui-components: 0.29.0(react@19.2.7) - instantsearch.js: 4.100.0(algoliasearch@5.53.0) - react: 19.2.7 - react-dom: 19.2.7(react@19.2.7) - react-instantsearch-core: 7.34.0(algoliasearch@5.53.0)(react@19.2.7) - react-is@16.13.1: {} react-markdown@10.1.0(@types/react@19.2.16)(react@19.2.7): @@ -9351,8 +9004,6 @@ snapshots: dependencies: compute-scroll-into-view: 3.1.1 - search-insights@2.17.3: {} - semver@7.8.5: optional: true @@ -9447,34 +9098,6 @@ snapshots: short-unique-id@5.3.2: {} - side-channel-list@1.0.1: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - - side-channel-map@1.0.1: - dependencies: - call-bound: 1.0.4 - es-errors: 1.3.0 - get-intrinsic: 1.3.0 - object-inspect: 1.13.4 - - side-channel-weakmap@1.0.2: - dependencies: - call-bound: 1.0.4 - es-errors: 1.3.0 - get-intrinsic: 1.3.0 - object-inspect: 1.13.4 - side-channel-map: 1.0.1 - - side-channel@1.1.1: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - side-channel-list: 1.0.1 - side-channel-map: 1.0.1 - side-channel-weakmap: 1.0.2 - siginfo@2.0.0: {} signal-exit@4.1.0: {} diff --git a/scripts/sync-crawler-config.ts b/scripts/sync-crawler-config.ts deleted file mode 100644 index 62e550a29..000000000 --- a/scripts/sync-crawler-config.ts +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env npx tsx - -/** - * Push algolia/crawler-config.js to the Algolia Crawler, then trigger a reindex. - * - * Usage: - * pnpm sync-crawler-config - * - * `PATCH /1/crawlers/{id}/config` is a top-level merge, so this only sends the - * fields we manage. We deliberately do NOT send `apiKey`: the crawler's index - * write key stays whatever it is in the Algolia dashboard and is never touched - * here (the config file doesn't store it either). Updating config doesn't crawl - * on its own, so we POST /reindex afterwards to apply the new config. - * - * `initialIndexSettings` in the crawler config only applies when the crawler - * *creates* the index — it is not re-applied to an existing index on subsequent - * crawls. To keep the live production index in sync with the versioned settings, - * we also push them via the Search API's PUT /1/indexes/{name}/settings when - * ALGOLIA_ADMIN_API_KEY is available. Without that key, we skip with a warning - * and the crawler config still updates normally. - * - * Requires these env vars (provided as CI secrets/variables): - * ALGOLIA_CRAWLER_ID - * ALGOLIA_CRAWLER_USER_ID - * ALGOLIA_CRAWLER_API_KEY - * - * Optional: - * ALGOLIA_ADMIN_API_KEY (needed to apply index settings to existing indices) - */ - -import { Buffer } from "node:buffer"; -import { crawlerConfig } from "../algolia/crawler-config.js"; - -const CRAWLER_ID = process.env.ALGOLIA_CRAWLER_ID; -const USER_ID = process.env.ALGOLIA_CRAWLER_USER_ID; -const API_KEY = process.env.ALGOLIA_CRAWLER_API_KEY; -const ADMIN_API_KEY = process.env.ALGOLIA_ADMIN_API_KEY; - -if (!(CRAWLER_ID && USER_ID && API_KEY)) { - console.error( - "Missing ALGOLIA_CRAWLER_ID, ALGOLIA_CRAWLER_USER_ID, or ALGOLIA_CRAWLER_API_KEY." - ); - process.exit(1); -} - -const API_BASE = "https://crawler.algolia.com/api/1"; -const authHeader = `Basic ${Buffer.from(`${USER_ID}:${API_KEY}`).toString("base64")}`; - -// The Crawler API stores recordExtractor as { __type: "function", source }, so -// serialize each action's function to its source string. Spreading crawlerConfig -// keeps every other managed field (schedule, sitemaps, actions, settings, …); -// appId is a harmless no-op in a merge and apiKey is intentionally absent. -const payload = { - ...crawlerConfig, - actions: crawlerConfig.actions.map((action) => ({ - ...action, - recordExtractor: { - __type: "function", - source: action.recordExtractor.toString(), - }, - })), -}; - -const configResponse = await fetch(`${API_BASE}/crawlers/${CRAWLER_ID}/config`, { - method: "PATCH", - headers: { Authorization: authHeader, "Content-Type": "application/json" }, - body: JSON.stringify(payload), -}); - -if (!configResponse.ok) { - console.error( - `Config update failed (${configResponse.status}): ${await configResponse.text()}` - ); - process.exit(1); -} -console.log("Crawler config updated."); - -// Push initialIndexSettings to each live index via the Search API. The crawler -// only applies these when the index is first created, so a settings change -// (e.g. removeWordsIfNoResults) never reaches an existing production index -// otherwise, and no-result rates stay high for compound agent queries. -if (ADMIN_API_KEY) { - const settingsByIndex = crawlerConfig.initialIndexSettings ?? {}; - for (const [indexName, settings] of Object.entries(settingsByIndex)) { - const response = await fetch( - `https://${crawlerConfig.appId}-dsn.algolia.net/1/indexes/${encodeURIComponent(indexName)}/settings`, - { - method: "PUT", - headers: { - "X-Algolia-Application-Id": crawlerConfig.appId, - "X-Algolia-API-Key": ADMIN_API_KEY, - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - } - ); - if (!response.ok) { - console.error( - `Index settings update failed for ${indexName} (${response.status}): ${await response.text()}` - ); - process.exit(1); - } - console.log(`Index settings pushed to ${indexName}.`); - } -} else { - console.warn( - "ALGOLIA_ADMIN_API_KEY not set — skipping Search API settings push. " + - "The crawler's initialIndexSettings only apply on index creation, so " + - "changes to those fields will not reach the existing production index " + - "until this key is provided. See algolia/README.md." - ); -} - -if (process.env.SKIP_REINDEX === "true") { - console.log("Skipping reindex — will be triggered after deployment."); - process.exit(0); -} - -const reindexResponse = await fetch( - `${API_BASE}/crawlers/${CRAWLER_ID}/reindex`, - { method: "POST", headers: { Authorization: authHeader } } -); - -if (!reindexResponse.ok) { - console.error( - `Reindex failed (${reindexResponse.status}): ${await reindexResponse.text()}` - ); - process.exit(1); -} - -const { taskId } = (await reindexResponse.json()) as { taskId: string }; -console.log(`Reindex triggered (taskId: ${taskId}).`); diff --git a/styles/config/vocabularies/Arcade/accept.txt b/styles/config/vocabularies/Arcade/accept.txt index 568ddd3f8..255e629fe 100644 --- a/styles/config/vocabularies/Arcade/accept.txt +++ b/styles/config/vocabularies/Arcade/accept.txt @@ -41,3 +41,5 @@ repo /ml-\d+/ /mr-\d+/ Clerk +Algolia +BM25 diff --git a/tests/search-bm25.test.ts b/tests/search-bm25.test.ts new file mode 100644 index 000000000..d7e56e042 --- /dev/null +++ b/tests/search-bm25.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; +import { createBm25Index } from "@/app/_lib/search/bm25"; +import { highlightQuery } from "@/app/_lib/search/highlight"; +import { documentsFromMdx } from "@/app/_lib/search/mdx-documents"; +import { tokenize } from "@/app/_lib/search/tokenize"; +import type { SearchDocument } from "@/app/_lib/search/types"; + +const doc = ( + overrides: Partial & Pick +): SearchDocument => ({ + url: overrides.url ?? overrides.id, + heading: overrides.heading ?? null, + content: overrides.content ?? "", + type: overrides.type ?? "page", + ...overrides, +}); + +describe("tokenize", () => { + it("splits camelCase, dotted names, and snake_case", () => { + expect(tokenize("Github.CreateIssue")).toEqual([ + "github", + "create", + "issue", + ]); + expect(tokenize("arcade_api_key")).toEqual(["arcade", "api", "key"]); + }); + + it("drops single-character tokens", () => { + expect(tokenize("a key")).toEqual(["key"]); + }); +}); + +describe("highlightQuery", () => { + it("marks query tokens inside the display text", () => { + const parts = highlightQuery("Getting Your API Key", "api key"); + const matched = parts.filter((part) => part.match).map((part) => part.text); + expect(matched).toEqual(["API", "Key"]); + }); + + it("returns the original text when the query is empty", () => { + expect(highlightQuery("Arcade", "")).toEqual([ + { text: "Arcade", match: false }, + ]); + }); +}); + +describe("createBm25Index", () => { + const corpus: SearchDocument[] = [ + doc({ + id: "/en/get-started/setup/api-keys", + title: "Getting Your API Key", + content: "Generate an Arcade API key from the dashboard or CLI.", + }), + doc({ + id: "/en/resources/integrations/development/github", + title: "GitHub", + content: "Repository and collaboration tools for agents.", + }), + doc({ + id: "/en/resources/integrations/development/github#githubcreateissue", + title: "GitHub", + heading: "Github.CreateIssue", + content: "Create a new issue in a GitHub repository.", + type: "tool", + }), + doc({ + id: "/en/operate/deploy", + title: "Deploy Arcade", + content: "Host the Arcade Engine on your own infrastructure.", + }), + ]; + + const index = createBm25Index(corpus); + + it("returns no hits for an empty query", () => { + expect(index.search("")).toEqual([]); + expect(index.search(" ")).toEqual([]); + }); + + it("ranks a title match above a content-only mention", () => { + const hits = index.search("api key"); + expect(hits.length).toBeGreaterThan(0); + expect(hits[0].id).toBe("/en/get-started/setup/api-keys"); + }); + + it("keeps the best record per page so a tool hash can win", () => { + const hits = index.search("create issue"); + const github = hits.find((hit) => + hit.url.startsWith("/en/resources/integrations/development/github") + ); + expect(github?.heading).toBe("Github.CreateIssue"); + expect(github?.url).toContain("#"); + }); + + it("caps the number of hits", () => { + expect(index.search("arcade", 1)).toHaveLength(1); + }); +}); + +describe("documentsFromMdx", () => { + it("emits a page record and heading records with anchors", () => { + const source = `--- +title: "Getting Your API Key" +description: "Learn how to obtain and manage your Arcade API key" +--- + +import { Steps } from "nextra/components"; + +# Getting Your API Key + +Before you begin, generate an Arcade API key. + +## Using the Dashboard + +Visit the API Keys page in Arcade Dashboard. + +## Using the CLI + +Run \`arcade login\` then create a key. +`; + + const documents = documentsFromMdx( + "/en/get-started/setup/api-keys", + source + ); + + expect(documents[0]).toMatchObject({ + type: "page", + title: "Getting Your API Key", + url: "/en/get-started/setup/api-keys", + }); + expect(documents[0].content).toContain("Learn how to obtain"); + + const dashboard = documents.find( + (document) => document.heading === "Using the Dashboard" + ); + expect(dashboard?.url).toBe( + "/en/get-started/setup/api-keys#using-the-dashboard" + ); + expect(dashboard?.content).toContain("API Keys page"); + expect( + documents.some((document) => document.content.includes("import")) + ).toBe(false); + }); +}); diff --git a/tests/search-index.test.ts b/tests/search-index.test.ts new file mode 100644 index 000000000..7b12e87fb --- /dev/null +++ b/tests/search-index.test.ts @@ -0,0 +1,166 @@ +import { copyFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createBm25Index } from "@/app/_lib/search/bm25"; +import { buildSearchIndex } from "@/app/_lib/search/build-index"; +import { documentsFromToolkit } from "@/app/_lib/search/toolkit-documents"; +import { readToolkitData } from "@/app/_lib/toolkit-data"; + +const fixtureToolkit = new URL( + "../toolkit-docs-generator/tests/fixtures/github-toolkit.json", + import.meta.url +); + +const tempDirs: string[] = []; + +const makeTempDir = async (): Promise => { + const dir = await mkdtemp(join(tmpdir(), "search-index-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ); +}); + +describe("documentsFromToolkit", () => { + it("indexes the toolkit page and each tool with matching anchors", async () => { + const dataDir = await makeTempDir(); + await copyFile(fixtureToolkit, join(dataDir, "github.json")); + const toolkit = await readToolkitData("Github", { dataDir }); + expect(toolkit).not.toBeNull(); + if (!toolkit) { + return; + } + + const documents = documentsFromToolkit(toolkit); + const page = documents.find((document) => document.type === "page"); + expect(page?.title).toBe("GitHub"); + expect(page?.url).toBe("/en/resources/integrations/development/github"); + + const createIssue = documents.find( + (document) => document.heading === "Github.CreateIssue" + ); + expect(createIssue?.type).toBe("tool"); + expect(createIssue?.url).toBe( + "/en/resources/integrations/development/github#githubcreateissue" + ); + expect(createIssue?.content).toContain("Create a new issue"); + }); + + it("skips hidden toolkits", async () => { + const dataDir = await makeTempDir(); + await copyFile(fixtureToolkit, join(dataDir, "github.json")); + const toolkit = await readToolkitData("Github", { dataDir }); + expect(toolkit).not.toBeNull(); + if (!toolkit) { + return; + } + + const hidden = { + ...toolkit, + metadata: { ...toolkit.metadata, isHidden: true }, + }; + expect(documentsFromToolkit(hidden)).toEqual([]); + }); +}); + +describe("buildSearchIndex", () => { + it("indexes authored MDX and skips dynamic-route templates", async () => { + const pagesDir = await makeTempDir(); + const dataDir = await makeTempDir(); + await writeFile( + join(pagesDir, "page.mdx"), + `--- +title: "Home" +description: "Arcade docs home" +--- + +# Home + +Welcome to Arcade. +`, + "utf8" + ); + + const documents = await buildSearchIndex({ pagesDir, dataDir }); + expect(documents.some((document) => document.title === "Home")).toBe(true); + expect( + documents.some((document) => document.url.includes("[toolkitId]")) + ).toBe(false); + }); + + it("lets authored MDX win the page record when it occupies a toolkit URL", async () => { + const pagesDir = await makeTempDir(); + const dataDir = await makeTempDir(); + const partnerDir = join( + pagesDir, + "resources", + "integrations", + "search", + "tavily" + ); + await mkdir(partnerDir, { recursive: true }); + await writeFile( + join(partnerDir, "page.mdx"), + `--- +title: "Tavily" +description: "Partner search MCP server" +--- + +# Tavily + +Enable agents to search the web. +`, + "utf8" + ); + await copyFile(fixtureToolkit, join(dataDir, "github.json")); + + const documents = await buildSearchIndex({ pagesDir, dataDir }); + const tavilyPages = documents.filter( + (document) => + document.type === "page" && + document.url === "/en/resources/integrations/search/tavily" + ); + expect(tavilyPages).toHaveLength(1); + expect(tavilyPages[0].title).toBe("Tavily"); + expect( + documents.some((document) => document.heading === "Github.CreateIssue") + ).toBe(true); + }); + + it("builds a corpus from the live docs tree that BM25 can search", async () => { + const documents = await buildSearchIndex(); + expect(documents.length).toBeGreaterThan(100); + + expect( + documents.some( + (document) => document.url === "/en/get-started/setup/api-keys" + ) + ).toBe(true); + expect( + documents.some( + (document) => + document.url === "/en/resources/integrations/development/github" + ) + ).toBe(true); + expect(documents.some((document) => document.type === "tool")).toBe(true); + expect( + documents.some((document) => document.url.includes("[toolkitId]")) + ).toBe(false); + + const index = createBm25Index(documents); + const apiKeyHits = index.search("getting your api key"); + expect(apiKeyHits[0]?.url).toContain("/en/get-started/setup/api-keys"); + + const githubHits = index.search("github create issue"); + expect( + githubHits.some((hit) => + hit.url.startsWith("/en/resources/integrations/development/github") + ) + ).toBe(true); + }); +}); diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index e66401cac..2c553688b 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -127,14 +127,11 @@ statically render the toolkit routes at build time from the committed JSON. ## Search indexing -Search uses an external Algolia crawler. There is no Pagefind or local search -index build step in this repository. After deployment, the crawler indexes the -rendered site. `app/_components/algolia-search.tsx` queries that index with the -public, read-only values configured through these Vercel environment variables: - -- `NEXT_PUBLIC_ALGOLIA_APP_ID` -- `NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY` -- `NEXT_PUBLIC_ALGOLIA_INDEX_NAME` +Search is an in-memory BM25 index built from authored MDX and generated +toolkit JSON at request/build time (`app/_lib/search/build-index.ts`). The +`/api/search-index` route serves the corpus; `app/_components/docs-search.tsx` +ranks hits in the browser. Preview deployments search the current branch — +there is no external crawler or search API key. ## Key files