diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82068f8fc7..c9f5160000 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,11 +68,11 @@ repos: ^c/include/.*[.](h|hpp)$| ^cpp/include/.*[.](h|hpp|cuh)$| ^python/cuvs/cuvs/.*[.](py|pyx|pxd)$| - ^java/cuvs-java/src/main/.*[.]java$| + ^java/cuvs-(java|lucene)/src/main/.*[.]java$| ^rust/cuvs/src/.*[.]rs$| ^go/.*[.]go$| ^fern/scripts/generate_api_reference[.]py$| - ^fern/pages/(c|cpp|python|java|rust|go)_api/.*[.]md$| + ^fern/pages/(c|cpp|python|java|lucene|rust|go)_api/.*[.]md$| ^fern/pages/api_reference[.]md$| ^fern/docs[.]yml$ - id: no-deprecationwarning diff --git a/fern/docs.yml b/fern/docs.yml index 9e47025b33..8963c518a0 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -544,6 +544,61 @@ navigation: path: "./pages/java_api/java-api-com-nvidia-cuvs-tieredindexparams.md" - page: "TieredIndexQuery" path: "./pages/java_api/java-api-com-nvidia-cuvs-tieredindexquery.md" + - section: "Lucene API Documentation" + path: "./pages/lucene_api/index.md" + contents: + - page: "AcceleratedHNSWParams" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams.md" + - page: "AcceleratedHNSWUtils" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils.md" + - page: "CagraIndexParamsFactory" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory.md" + - page: "FilterBitsetCacheConfig" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig.md" + - page: "FilterCuVSServiceProvider" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider.md" + - page: "GPUIndex" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuindex.md" + - page: "GPUSearchParams" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams.md" + - page: "LuceneProvider" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneprovider.md" + - page: "ThreadLocalCuVSResourcesProvider" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider.md" + - page: "Utils" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-utils.md" + - page: "CuVS2510GPUSearchCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md" + - page: "CuVS2510GPUVectorsFormat" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat.md" + - page: "CuVS2510GPUVectorsReader" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md" + - page: "CuVS2510GPUVectorsWriter" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter.md" + - page: "FieldWriter" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-fieldwriter.md" + - page: "GPUBuiltHnswGraph" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph.md" + - page: "GPUKnnFloatVectorQuery" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md" + - page: "Lucene101AcceleratedHNSWCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md" + - page: "Lucene99AcceleratedHNSWVectorsFormat" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat.md" + - page: "Lucene99AcceleratedHNSWVectorsWriter" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter.md" + - page: "LuceneAcceleratedHNSWBinaryQuantizedCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md" + - page: "LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat.md" + - page: "LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter.md" + - page: "LuceneAcceleratedHNSWScalarQuantizedCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md" + - page: "LuceneAcceleratedHNSWScalarQuantizedVectorsFormat" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat.md" + - page: "LuceneAcceleratedHNSWScalarQuantizedVectorsWriter" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter.md" - section: "Rust API Documentation" path: "./pages/rust_api/index.md" contents: diff --git a/fern/pages/api_reference.md b/fern/pages/api_reference.md index f52a20d2b0..f57790a410 100644 --- a/fern/pages/api_reference.md +++ b/fern/pages/api_reference.md @@ -14,5 +14,6 @@ For task-focused examples and usage guidance, see the [API Guide](/user-guide/ap - [C++ API Documentation](/api-reference/cpp-api-documentation): inspect the core NVIDIA cuVS C++ APIs, including common RAFT types used in public headers. - [Python API Documentation](/api-reference/python-api-documentation): inspect Python modules, classes, functions, and parameters. - [Java API Documentation](/api-reference/java-api-documentation): inspect Java classes, resources, matrices, indexes, and query APIs. +- [Lucene API Documentation](/api-reference/lucene-api-documentation): inspect the Apache Lucene codecs that offload vector index build and search to the GPU. - [Rust API Documentation](/api-reference/rust-api-documentation): inspect Rust modules, structs, and wrappers around NVIDIA cuVS APIs. - [Go API Documentation](/api-reference/go-api-documentation): inspect Go packages for supported NVIDIA cuVS APIs. diff --git a/fern/pages/lucene_api/index.md b/fern/pages/lucene_api/index.md new file mode 100644 index 0000000000..69215257e7 --- /dev/null +++ b/fern/pages/lucene_api/index.md @@ -0,0 +1,35 @@ +# Lucene API Documentation + +These pages are generated from the Java source files in `java/cuvs-lucene/src/main`. + +## Common + +- [AcceleratedHNSWParams](/api-reference/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams) +- [AcceleratedHNSWUtils](/api-reference/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils) +- [CagraIndexParamsFactory](/api-reference/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory) +- [FilterBitsetCacheConfig](/api-reference/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig) +- [FilterCuVSServiceProvider](/api-reference/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider) +- [GPUIndex](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuindex) +- [GPUSearchParams](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams) +- [LuceneProvider](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneprovider) +- [ThreadLocalCuVSResourcesProvider](/api-reference/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider) +- [Utils](/api-reference/lucene-api-com-nvidia-cuvs-lucene-utils) + +## Codecs and Formats + +- [CuVS2510GPUSearchCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec) +- [CuVS2510GPUVectorsFormat](/api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat) +- [CuVS2510GPUVectorsReader](/api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader) +- [CuVS2510GPUVectorsWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter) +- [FieldWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-fieldwriter) +- [GPUBuiltHnswGraph](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph) +- [GPUKnnFloatVectorQuery](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery) +- [Lucene101AcceleratedHNSWCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec) +- [Lucene99AcceleratedHNSWVectorsFormat](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat) +- [Lucene99AcceleratedHNSWVectorsWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter) +- [LuceneAcceleratedHNSWBinaryQuantizedCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec) +- [LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat) +- [LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter) +- [LuceneAcceleratedHNSWScalarQuantizedCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec) +- [LuceneAcceleratedHNSWScalarQuantizedVectorsFormat](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat) +- [LuceneAcceleratedHNSWScalarQuantizedVectorsWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter) diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams.md new file mode 100644 index 0000000000..27ffe5cb39 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams.md @@ -0,0 +1,537 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswparams +--- + +# AcceleratedHNSWParams + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class AcceleratedHNSWParams +``` + +## Public Members + +### getWriterThreads + +```java +public int getWriterThreads() +``` + +Get the cuVS writer threads parameter + +**Returns** + +cuVS writer threads parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:149`_ + +### getIntermediateGraphDegree + +```java +public int getIntermediateGraphDegree() +``` + +Get the intermediate graph degree + +**Returns** + +the graph degree parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:158`_ + +### getGraphdegree + +```java +public int getGraphdegree() +``` + +Get the graph degree + +**Returns** + +the graph degree parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:167`_ + +### getHnswLayers + +```java +public int getHnswLayers() +``` + +Get the number of HNSW layers + +**Returns** + +the number of HNSW layers + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:176`_ + +### getMaxConn + +```java +public int getMaxConn() +``` + +Get the max connection parameter + +**Returns** + +the max connection parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:185`_ + +### getBeamWidth + +```java +public int getBeamWidth() +``` + +Get the beam width parameter + +**Returns** + +the beam width parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:194`_ + +### getCagraGraphBuildAlgo + +```java +public CagraGraphBuildAlgo getCagraGraphBuildAlgo() +``` + +Get the CAGRA graph build algorithm + +**Returns** + +the CAGRA graph build algorithm + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:203`_ + +### getCuVSIvfPqParams + +```java +public CuVSIvfPqParams getCuVSIvfPqParams() +``` + +Get the instance of `CuVSIvfPqParams` + +**Returns** + +the instance of `CuVSIvfPqParams` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:212`_ + +### getNumMergeWorkers + +```java +public int getNumMergeWorkers() +``` + +Get the number of merge workers set to be used in the fallback mechanism + +**Returns** + +the number of merge workers + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:221`_ + +### getMergeExec + +```java +public ExecutorService getMergeExec() +``` + +Get the instance of the `ExecutorService` to be used in the fallback mechanism + +**Returns** + +the instance of the `ExecutorService` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:230`_ + +### getStrategy + +```java +public Strategy getStrategy() +``` + +Get the chosen strategy: + +When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set +When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + +**Returns** + +get the chosen `Strategy` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:242`_ + +### getCuvsDistanceType + +```java +public CuvsDistanceType getCuvsDistanceType() +``` + +Get the cuvs distance type + +**Returns** + +the distance type + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:251`_ + +### getNNDescentNumIterations + +```java +public int getNNDescentNumIterations() +``` + +get the number of Iterations to run if building with NN_DESCENT + +**Returns** + +the number of iterations for NN_DESCENT + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:260`_ + +### getHnswHeuristicType + +```java +public HnswHeuristicType getHnswHeuristicType() +``` + +Get the heuristic cuVS applies when deriving the CAGRA build parameters from maxConn and +beamWidth. Only consulted under the `Strategy#HEURISTIC` strategy. + +**Returns** + +the `HnswHeuristicType` to hand to cuVS + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:270`_ + +### withWriterThreads + +```java +public Builder withWriterThreads(int writerThreads) +``` + +Set the number of cuVS writer threads while building the index +Valid range - Minimum: \{@value MIN_WRITER_THREADS\}, Maximum: \{@value MAX_WRITER_THREADS\} +Default value - \{@value DEFAULT_WRITER_THREADS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `writerThreads` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:335`_ + +### withIntermediateGraphDegree + +```java +public Builder withIntermediateGraphDegree(int intermediateGraphDegree) +``` + +Set the intermediate graph degree to use while building CAGRA index +Valid range - Minimum: \{@value MIN_INT_GRAPH_DEG\}, Maximum: \{@value MAX_INT_GRAPH_DEG\} +Default value - \{@value DEFAULT_INT_GRAPH_DEGREE\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `intermediateGraphDegree` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:348`_ + +### withGraphDegree + +```java +public Builder withGraphDegree(int graphDegree) +``` + +Set the graph degree to use while building CAGRA index +Valid range - Minimum: \{@value MIN_GRAPH_DEG\}, Maximum: \{@value MAX_GRAPH_DEG\} +Default value - \{@value DEFAULT_GRAPH_DEGREE\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `graphDegree` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:361`_ + +### withHNSWLayer + +```java +public Builder withHNSWLayer(int hnswLayers) +``` + +Set the number of HNSW layers to construct while building the HNSW index +Valid range - Minimum: \{@value MIN_HNSW_LAYERS\}, Maximum: \{@value MAX_HNSW_LAYERS\} +Default value - \{@value DEFAULT_HNSW_LAYERS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `hnswLayers` | the number of HNSW layers | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:374`_ + +### withMaxConn + +```java +public Builder withMaxConn(int maxConn) +``` + +Set the max connections parameter while building HNSW index with fallback mechanism +Valid range - Minimum: \{@value MIN_MAX_CONN\}, Maximum: \{@value MAX_MAX_CONN\} +Default value - \{@value DEFAULT_MAX_CONN\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `maxConn` | the max connections parameter | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:387`_ + +### withBeamWidth + +```java +public Builder withBeamWidth(int beamWidth) +``` + +Set the beam width parameter while building HNSW index with fallback mechanism +Valid range - Minimum: \{@value MIN_BEAM_WIDTH\}, Maximum: \{@value MAX_BEAM_WIDTH\} +Default value - \{@value DEFAULT_BEAM_WIDTH\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `beamWidth` | the beam width parameter | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:400`_ + +### withCagraGraphBuildAlgo + +```java +public Builder withCagraGraphBuildAlgo(CagraGraphBuildAlgo cagraGraphBuildAlgo) +``` + +Set the CAGRA graph build algorithm to use +Default value - NN_DESCENT + +**Parameters** + +| Name | Description | +| --- | --- | +| `cagraGraphBuildAlgo` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:412`_ + +### withCuVSIvfPqParams + +```java +public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) +``` + +Set the instance of `CuVSIvfPqParams` + +**Parameters** + +| Name | Description | +| --- | --- | +| `cuVSIvfPqParams` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:423`_ + +### withNumMergeWorkers + +```java +public Builder withNumMergeWorkers(int numMergeWorkers) +``` + +Set the number of merge workers to be used with the fallback mechanism +Default value - \{@value DEFAULT_NUM_MERGE_WORKERS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `numMergeWorkers` | number of merge workers to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:435`_ + +### withMergeExecutorService + +```java +public Builder withMergeExecutorService(ExecutorService mergeExec) +``` + +Set the merge executor service to be used in the fallback mechanism +Default value an instance with one thread + +**Parameters** + +| Name | Description | +| --- | --- | +| `mergeExec` | an instance of `ExecutorService` | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:447`_ + +### withStrategy + +```java +public Builder withStrategy(Strategy strategy) +``` + +Set the chosen strategy: + +When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set +When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + +Valid options - HEURISTIC, CUSTOM +Default value - HEURISTIC + +**Parameters** + +| Name | Description | +| --- | --- | +| `strategy` | , the strategy to choose | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:464`_ + +### withCuvsDistanceType + +```java +public Builder withCuvsDistanceType(CuvsDistanceType cuvsDistanceType) +``` + +Set the CuvsDistanceType + +**Parameters** + +| Name | Description | +| --- | --- | +| `cuvsDistanceType` | the CuvsDistanceType to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:475`_ + +### withNNDescentNumIterations + +```java +public Builder withNNDescentNumIterations(int nnDescentNumIterations) +``` + +Set the number of Iterations to run if building with NN_DESCENT + +Valid range - Minimum: \{@value MIN_NN_DESCENT_NUM_ITERATIONS\}, Maximum: \{@value MAX_NN_DESCENT_NUM_ITERATIONS\} +Default value - \{@value DEFAULT_NN_DESCENT_NUM_ITERATIONS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `nnDescentNumIterations` | number of merge workers to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:489`_ + +### withHnswHeuristicType + +```java +public Builder withHnswHeuristicType(HnswHeuristicType hnswHeuristicType) +``` + +Set the heuristic cuVS applies when deriving the CAGRA build parameters from maxConn and +beamWidth. Only consulted under the `Strategy#HEURISTIC` strategy. + +Default value - SAME_GRAPH_FOOTPRINT, which targets a CAGRA graph of the same on-disk size as +the equivalent HNSW graph (graph degree = 2 * maxConn). + +**Parameters** + +| Name | Description | +| --- | --- | +| `hnswHeuristicType` | the `HnswHeuristicType` to hand to cuVS | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:504`_ + +### build + +```java +public AcceleratedHNSWParams build() +``` + +Create an instance of `AcceleratedHNSWParams` + +**Returns** + +instance of `AcceleratedHNSWParams` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:600`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java:17`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils.md new file mode 100644 index 0000000000..219149bc6d --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils.md @@ -0,0 +1,177 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils +--- + +# AcceleratedHNSWUtils + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class AcceleratedHNSWUtils +``` + +## Public Members + +### createSingleVectorHnswGraph + +```java +public static GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimensions) throws Throwable +``` + +Creates a dummy HNSW graph for a single vector. +The graph will have 1 level with 1 node and no neighbors. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:54`_ + +### createMultiLayerHnswGraph + +```java +public static GPUBuiltHnswGraph createMultiLayerHnswGraph( FieldInfo fieldInfo, int size, int dimensions, CuVSMatrix adjacencyListMatrix, List vectors, int hnswLayers, CagraIndexParams params, QuantizationType quantization) throws Throwable +``` + +Creates a multi-layer HNSW graph with dynamic number of layers. +M = ceil(cagraGraphDegree / 2), where cagraGraphDegree is the CAGRA adjacency list's degree +(its column count). Ceil is used to accommodate odd graph degrees. +Each layer contains 1/M nodes from the previous layer +Creates layers until the highest layer has ≤ M nodes + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:81`_ + +### writeGraph + +```java +public static int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) throws IOException +``` + +Returns a 2D array of offsets (information written while writing the meta info) + +**Parameters** + +| Name | Description | +| --- | --- | +| `graph` | instance of GPUBuiltHnswGraph | +| `vectorIndex` | instance of IndexOutput | + +**Returns** + +a 2D array of offsets + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O Exceptions | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:237`_ + +### writeMeta + +```java +public static void writeMeta( IndexOutput vectorIndex, IndexOutput meta, FieldInfo field, long vectorIndexOffset, long vectorIndexLength, int count, HnswGraph graph, int[][] graphLevelNodeOffsets) throws IOException +``` + +Writes the meta information for the index. + +**Parameters** + +| Name | Description | +| --- | --- | +| `vectorIndex` | instance of IndexOutput | +| `meta` | instance of IndexOutput | +| `field` | instance of FieldInfo | +| `vectorIndexOffset` | vector index offset | +| `vectorIndexLength` | vector index length | +| `count` | the count of vectors | +| `graph` | instance of HnswGraph | +| `graphLevelNodeOffsets` | graph level node offsets | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O Exceptions | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:302`_ + +### printInfoStream + +```java +public static void printInfoStream(InfoStream infoStream, String component, String msg) +``` + +A utility method to print info/debugging messages using InfoStream. + +**Parameters** + +| Name | Description | +| --- | --- | +| `msg` | the debugging message to print | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:384`_ + +### writeEmpty + +```java +public static void writeEmpty(FieldInfo fieldInfo, IndexOutput op) throws IOException +``` + +Writes an empty meta information for the field. + +**Parameters** + +| Name | Description | +| --- | --- | +| `fieldInfo` | instance of FieldInfo | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O Exceptions | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:396`_ + +### quantizeFloatVectorsToBinary + +```java +public static List quantizeFloatVectorsToBinary(List floatVectors) +``` + +Quantizes FLOAT32 vectors to binary (1 bit per dimension, packed into bytes). +Binary quantization: each dimension is compared to a centroid (mean of all values for that dimension). +If value > centroid, bit = 1, else bit = 0. +Bits are packed: 8 dimensions per byte. + +**Parameters** + +| Name | Description | +| --- | --- | +| `floatVectors` | A list of float vectors | + +**Returns** + +A list of byte binary representation for the input vectors + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:409`_ + +### quantizeFloatVectorsToScalar + +```java +public static List quantizeFloatVectorsToScalar(List floatVectors) +``` + +Scalar quantization. + +**Parameters** + +| Name | Description | +| --- | --- | +| `floatVectors` | A list of float vectors | + +**Returns** + +A list of byte scalar representation for the input vectors + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:451`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java:31`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory.md new file mode 100644 index 0000000000..e6b3445de6 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory.md @@ -0,0 +1,66 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory +--- + +# CagraIndexParamsFactory + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class CagraIndexParamsFactory +``` + +A centralized place for producing `CagraIndexParams` from the cuvs-lucene input parameter +classes based on the chosen strategy. + +For the `HEURISTIC` strategy the build heuristics are delegated to cuVS. + +## Public Members + +### create + +```java +public static CagraIndexParams create( GPUSearchParams gpuSearchParams, long rows, long dimension) +``` + +Creates an instance of `CagraIndexParams` for the GPU-native CAGRA index based on the +chosen strategy in the `GPUSearchParams`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `gpuSearchParams` | the input parameters for the build and search on the GPU API | +| `rows` | number of vectors in the data set | +| `dimension` | the dimension of the vectors in the data set | + +**Returns** + +an instance of `CagraIndexParams` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java:29`_ + +### create + +```java +public static CagraIndexParams create( AcceleratedHNSWParams acceleratedHNSWParams, long rows, long dimension) +``` + +Creates an instance of `CagraIndexParams` for the accelerated-HNSW index based on the +chosen strategy in the `AcceleratedHNSWParams`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | the input parameters for the build on the GPU API | +| `rows` | number of vectors in the data set | +| `dimension` | the dimension of the vectors in the data set | + +**Returns** + +an instance of `CagraIndexParams` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java:73`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java:17`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md new file mode 100644 index 0000000000..c7120dc9d4 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md @@ -0,0 +1,147 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec +--- + +# CuVS2510GPUSearchCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class CuVS2510GPUSearchCodec extends FilterCodec +``` + +cuVS based codec for GPU based vector search that enables both - indexing and search on the GPU. +cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. + +## Public Members + +### CuVS2510GPUSearchCodec + +```java +public CuVS2510GPUSearchCodec() throws Exception +``` + +Default constructor for `CuVS2510GPUSearchCodec`. + +**Throws** + +| Type | Description | +| --- | --- | +| `Exception` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:30`_ + +### CuVS2510GPUSearchCodec + +```java +public CuVS2510GPUSearchCodec(String name, Codec delegate) +``` + +Initialize `CuVS2510GPUSearchCodec` with an instance of `GPUSearchParams` +having default parameter values. + +**Parameters** + +| Name | Description | +| --- | --- | +| `name` | the name of the codec | +| `delegate` | the delegate codec | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:45`_ + +### CuVS2510GPUSearchCodec + +```java +public CuVS2510GPUSearchCodec(GPUSearchParams params) throws Exception +``` + +Initialize the codec with an instance of `GPUSearchParams` having either default +or overridden parameter values. + +**Parameters** + +| Name | Description | +| --- | --- | +| `params` | An instance of `GPUSearchParams` | + +**Throws** + +| Type | Description | +| --- | --- | +| `Exception` | Exception raised when initializing the codec | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:56`_ + +### CuVS2510GPUSearchCodec + +```java +public CuVS2510GPUSearchCodec(GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) throws Exception +``` + +Initialize the codec with GPU search and filter-bitset-cache parameters. + +**Parameters** + +| Name | Description | +| --- | --- | +| `params` | GPU index and search parameters | +| `filterCacheConfig` | filter-bitset-cache configuration | + +**Throws** + +| Type | Description | +| --- | --- | +| `Exception` | Exception raised when initializing the codec | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:67`_ + +### CuVS2510GPUSearchCodec + +```java +public CuVS2510GPUSearchCodec( String name, Codec delegate, GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) +``` + +Initialize a named codec with explicit delegate, GPU search, and filter-cache parameters. + +**Parameters** + +| Name | Description | +| --- | --- | +| `name` | the name of the codec | +| `delegate` | the delegate codec | +| `params` | GPU index and search parameters | +| `filterCacheConfig` | filter-bitset-cache configuration | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:80`_ + +### knnVectorsFormat + +```java +@Override public KnnVectorsFormat knnVectorsFormat() +``` + +Get the configured `KnnVectorsFormat`. + +**Returns** + +the instance of the `KnnVectorsFormat` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:111`_ + +### setKnnFormat + +```java +public void setKnnFormat(KnnVectorsFormat format) +``` + +Set the `KnnVectorsFormat`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `format` | the `KnnVectorsFormat` to set | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:121`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:20`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat.md new file mode 100644 index 0000000000..b747d12460 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat.md @@ -0,0 +1,109 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsformat +--- + +# CuVS2510GPUVectorsFormat + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat +``` + +Extends upon the KnnVectorsFormat - Encodes/decodes per-document vector and any associated indexing structures required to support +GPU-based accelerated nearest-neighbor search. + +## Public Members + +### CuVS2510GPUVectorsFormat + +```java +public CuVS2510GPUVectorsFormat() +``` + +Initializes the `CuVS2510GPUVectorsFormat` with default parameter values. + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:55`_ + +### CuVS2510GPUVectorsFormat + +```java +public CuVS2510GPUVectorsFormat(GPUSearchParams gpuSearchParams) +``` + +Initializes the `CuVS2510GPUVectorsFormat` with an instance of `GPUSearchParams`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `gpuSearchParams` | An instance of `GPUSearchParams` | + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:65`_ + +### CuVS2510GPUVectorsFormat + +```java +public CuVS2510GPUVectorsFormat( GPUSearchParams gpuSearchParams, FilterBitsetCacheConfig filterCacheConfig) +``` + +Initializes the format with GPU search and filter-bitset-cache parameters. + +**Parameters** + +| Name | Description | +| --- | --- | +| `gpuSearchParams` | GPU index and search parameters | +| `filterCacheConfig` | filter-bitset-cache configuration | + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:76`_ + +### fieldsWriter + +```java +@Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException +``` + +Returns a KnnVectorsReader instance to write the vectors to the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:86`_ + +### fieldsReader + +```java +@Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException +``` + +Returns a KnnVectorsReader instance to read the vectors from the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:96`_ + +### getMaxDimensions + +```java +@Override public int getMaxDimensions(String fieldName) +``` + +Returns the maximum number of vector dimensions supported by this codec for the given field name. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:106`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java:25`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md new file mode 100644 index 0000000000..99f4c8062b --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md @@ -0,0 +1,335 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader +--- + +# CuVS2510GPUVectorsReader + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class CuVS2510GPUVectorsReader extends KnnVectorsReader +``` + +KnnVectorsReader instance associated with cuVS format for reading vectors from an index. + +## Public Members + +### CuVS2510GPUVectorsReader + +```java +public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatReader) throws IOException +``` + +Initializes the `CuVS2510GPUVectorsReader`, checks and loads the index. + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the SegmentReadState | +| `flatReader` | instance of the FlatVectorsReader | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O exception | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:87`_ + +### CuVS2510GPUVectorsReader + +```java +CuVS2510GPUVectorsReader( SegmentReadState state, FlatVectorsReader flatReader, FilterBitsetCache filterBitsetCache) throws IOException +``` + +Initializes the reader with the cache owned by its vectors format. + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the SegmentReadState | +| `flatReader` | instance of the FlatVectorsReader | +| `filterBitsetCache` | filter cache shared by readers from the same vectors format | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O exception | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:100`_ + +### readSimilarityFunction + +```java +static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws IOException +``` + +Checks the distance function validity and returns it. + +**Parameters** + +| Name | Description | +| --- | --- | +| `input` | instance of DataInput | + +**Returns** + +an instance of VectorSimilarityFunction + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:228`_ + +### readVectorEncoding + +```java +static VectorEncoding readVectorEncoding(DataInput input) throws IOException +``` + +Reads the vector encoding (The numeric data type of the vector values) from the DataInput. + +**Parameters** + +| Name | Description | +| --- | --- | +| `input` | instance of DataInput | + +**Returns** + +the vector encoding + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:243`_ + +### getFieldEntry + +```java +FieldEntry getFieldEntry(String field) +``` + +Gets the FieldEntry for the given field name, or `null` when this segment holds no +cuVS index for it. + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | name of the field | + +**Returns** + +the meta information for the field, or `null` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:306`_ + +### openCagraIndexForMerge + +```java +CagraIndex openCagraIndexForMerge(String field) throws IOException +``` + +Deserializes the CAGRA index of a single field onto the GPU, bypassing the `GPUIndex` +cache. This is how `CuVS2510GPUVectorsWriter` obtains the inputs for the cuVS merge API: +a reader opened with `Context#MERGE` has no cached indexes at all, and a cached index +belongs to the `com.nvidia.cuvs.CuVSResources` of whichever thread opened the reader, +while the merge API requires every input to share one resources instance - the merging +thread's. + +The returned index is owned by the caller and must be closed by it. + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | name of the field | + +**Returns** + +a freshly loaded CAGRA index, or `null` if this segment has none for the field + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O exception | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:325`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Closes the resources. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:394`_ + +### checkIntegrity + +```java +@Override public void checkIntegrity() throws IOException +``` + +Checks consistency of this reader. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:412`_ + +### getFloatVectorValues + +```java +@Override public FloatVectorValues getFloatVectorValues(String field) throws IOException +``` + +Returns the FloatVectorValues for the given field. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:421`_ + +### getByteVectorValues + +```java +@Override public ByteVectorValues getByteVectorValues(String field) +``` + +Returns the ByteVectorValues for the given field. + +This is not supported. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:431`_ + +### search + +```java +@Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException +``` + +Returns the k nearest neighbor documents using cuVS's CAGRA or brute force algorithm for this field, to the given vector. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:455`_ + +### search + +```java +@Override public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException +``` + +Return the k nearest neighbor documents as determined by comparison of their vector values for this field, to the given vector. + +This is not supported. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:598`_ + +### readEntry + +```java +static FieldEntry readEntry( IndexInput input, VectorEncoding vectorEncoding, VectorSimilarityFunction similarityFunction) throws IOException +``` + +Returns an instance of FieldEntry. + +**Parameters** + +| Name | Description | +| --- | --- | +| `input` | instance of IndexInput | +| `vectorEncoding` | The numeric data type of the vector values | +| `similarityFunction` | Vector similarity function; used in search to return top K most similar vectors to a target vector | + +**Returns** + +an instance of FieldEntry + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O Exceptions | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:626`_ + +### getCagraIndexForField + +```java +public CagraIndex getCagraIndexForField(String field) +``` + +Returns the `CagraIndex` for the given field, or `null` if unavailable +(e.g., during a merge or when the field is missing). + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | the vector field name | + +**Returns** + +the CAGRA index, or `null` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:678`_ + +### getFilterBitsetCache + +```java +FilterBitsetCache getFilterBitsetCache() +``` + +Returns the filter cache owned by the vectors format that created this reader. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:688`_ + +### getFieldInfos + +```java +public FieldInfos getFieldInfos() +``` + +Gets the instance of FieldInfos. + +**Returns** + +the instance of FieldInfos + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:697`_ + +### getCuvsIndexes + +```java +public IntObjectHashMap getCuvsIndexes() +``` + +Gets the map of `GPUIndex` objects. + +**Returns** + +the map of GPU index objects + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:706`_ + +### getFieldEntries + +```java +public IntObjectHashMap getFieldEntries() +``` + +Gets the map of FieldEntry objects that hold the meta information for the field. + +**Returns** + +the map of FieldEntry objects + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:715`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:59`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter.md new file mode 100644 index 0000000000..df1abbbdc7 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter.md @@ -0,0 +1,131 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorswriter +--- + +# CuVS2510GPUVectorsWriter + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter +``` + +extends upon KnnVectorsWriter and has implementation for critical methods like flush, merge etc. + +## Public Members + +### CAGRA + +```java +CAGRA(true, false), /** Builds a Brute Force index. */ BRUTE_FORCE(false, true), /** Builds both - CAGRA and Brute Force indexes. */ CAGRA_AND_BRUTE_FORCE(true, true) +``` + +Builds a CAGRA index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:89`_ + +### BRUTE_FORCE + +```java +BRUTE_FORCE(false, true), /** Builds both - CAGRA and Brute Force indexes. */ CAGRA_AND_BRUTE_FORCE(true, true) +``` + +Builds a Brute Force index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:92`_ + +### CAGRA_AND_BRUTE_FORCE + +```java +CAGRA_AND_BRUTE_FORCE(true, true) +``` + +Builds both - CAGRA and Brute Force indexes. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:95`_ + +### CuVS2510GPUVectorsWriter + +```java +public CuVS2510GPUVectorsWriter( SegmentWriteState state, GPUSearchParams gpuSearchParams, FlatVectorsWriter flatVectorsWriter) throws IOException +``` + +Initializes `CuVS2510GPUVectorsWriter`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the SegmentWriteState | +| `gpuSearchParams` | An instance of `GPUSearchParams` | +| `flatVectorsWriter` | instance of FlatVectorsWriter | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O exceptions | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:121`_ + +### addField + +```java +@Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException +``` + +Add new field for indexing. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:160`_ + +### flush + +```java +@Override public void flush(int maxDoc, DocMap sortMap) throws IOException +``` + +Creates the CAGRA and/or brute force indexes and writes them to the disk. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:303`_ + +### mergeOneField + +```java +@Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException +``` + +Write field for merging. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:685`_ + +### ramBytesUsed + +```java +@Override public long ramBytesUsed() +``` + +Returns the memory usage of this object in bytes. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:696`_ + +### finish + +```java +@Override public void finish() throws IOException +``` + +Called once at the end before close. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:708`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Close the applicable resources. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:728`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java:59`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-fieldwriter.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-fieldwriter.md new file mode 100644 index 0000000000..2e735241dd --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-fieldwriter.md @@ -0,0 +1,13 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-fieldwriter +--- + +# FieldWriter + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class FieldWriter extends KnnFieldVectorsWriter +``` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java:20`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig.md new file mode 100644 index 0000000000..3f215f360c --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig.md @@ -0,0 +1,42 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig +--- + +# FilterBitsetCacheConfig + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public record FilterBitsetCacheConfig(boolean enabled, long maxBytes) +``` + +Configuration for the filter-bitset cache used by multi-segment GPU searches. + +This configuration is local to one vectors-format instance. It does not configure cuvs-java's +process-wide filter-bitset device pool. + +The byte budget is a retention cap, not a preallocation. Packed host arrays and their +device-side mirrors are allocated as cache entries are populated. Actual host usage can exceed +the accounted payload bytes because of object overhead, temporary filter construction, and +in-flight handles. + +**Parameters** + +| Name | Description | +| --- | --- | +| `enabled` | whether filter bitsets are cached between searches | +| `maxBytes` | maximum bytes cached by this format; must be positive when caching is enabled | + +## Public Members + +### FilterBitsetCacheConfig + +```java +public static final FilterBitsetCacheConfig DEFAULT = new FilterBitsetCacheConfig(true, DEFAULT_MAX_BYTES) +``` + +Default configuration used by SPI-created codecs and vector formats. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterBitsetCacheConfig.java:27`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterBitsetCacheConfig.java:22`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider.md new file mode 100644 index 0000000000..9d6701b88e --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider.md @@ -0,0 +1,27 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-filtercuvsserviceprovider +--- + +# FilterCuVSServiceProvider + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class FilterCuVSServiceProvider extends CuVSServiceProvider +``` + +A provider that creates instances of FilterCuVSProvider. + +## Public Members + +### get + +```java +@Override public CuVSProvider get(CuVSProvider builtinProvider) +``` + +Initialize and return an CuVSProvider provided by this provider. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java:19`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java:15`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph.md new file mode 100644 index 0000000000..f8a3aee9dc --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph.md @@ -0,0 +1,155 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph +--- + +# GPUBuiltHnswGraph + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class GPUBuiltHnswGraph extends HnswGraph +``` + +This class holds the in-memory representation of the HNSW graph + +## Public Members + +### GPUBuiltHnswGraph + +```java +public GPUBuiltHnswGraph( int size, int dimensions, List layerNodes, List layerAdjacencies) +``` + +Multi-layer constructor that supports arbitrary number of layers. + +**Parameters** + +| Name | Description | +| --- | --- | +| `size` | the size of the dataset | +| `dimensions` | the vector dimension | +| `layerNodes` | the nodes on the layer | +| `layerAdjacencies` | adjacency list | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:41`_ + +### getNodesOnLevel + +```java +public NodesIterator getNodesOnLevel(int level) +``` + +Get all nodes on a given level as node 0th ordinals. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:89`_ + +### getNeighbors + +```java +public NeighborArray getNeighbors(int level, int node) +``` + +Get the neighbors for the node and the level it resides. + +**Parameters** + +| Name | Description | +| --- | --- | +| `level` | the level | +| `node` | the node | + +**Returns** + +an instance of NeighborArray + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:107`_ + +### seek + +```java +@Override public void seek(int level, int target) +``` + +Move the pointer to exactly the given level's target. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:132`_ + +### nextNeighbor + +```java +@Override public int nextNeighbor() +``` + +Iterates over the neighbor list. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:142`_ + +### entryNode + +```java +@Override public int entryNode() +``` + +Returns graph's entry point on the top level. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:173`_ + +### maxConn + +```java +@Override public int maxConn() +``` + +returns M, the maximum number of connections for a node. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:192`_ + +### neighborCount + +```java +@Override public int neighborCount() +``` + +Returns the neighbor count. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:207`_ + +### size + +```java +public int size() +``` + +Returns the number of nodes in the graph. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:282`_ + +### numLevels + +```java +public int numLevels() +``` + +Returns the number of levels in the HNSW graph. + +**Returns** + +the number of levels + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:291`_ + +### dimensions + +```java +public int dimensions() +``` + +Gets the vector dimension. + +**Returns** + +the vector dimension + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:300`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java:21`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuindex.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuindex.md new file mode 100644 index 0000000000..af188e002f --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuindex.md @@ -0,0 +1,134 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuindex +--- + +# GPUIndex + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class GPUIndex implements Closeable +``` + +This class holds references to the cuVS Index (Cagra, Brute force, etc.) + +## Public Members + +### GPUIndex + +```java +public GPUIndex( String segmentName, String fieldName, CagraIndex cagraIndex, int maxDocs, BruteForceIndex bruteforceIndex) +``` + +Initializes an instance of `GPUIndex` + +**Parameters** + +| Name | Description | +| --- | --- | +| `segmentName` | the name of the segment | +| `fieldName` | the field name | +| `cagraIndex` | reference to the CagraIndex | +| `maxDocs` | the maximum documents | +| `bruteforceIndex` | reference to the BruteForceIndex | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:35`_ + +### GPUIndex + +```java +public GPUIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex) +``` + +Initializes an instance of `GPUIndex` + +**Parameters** + +| Name | Description | +| --- | --- | +| `cagraIndex` | reference to the CagraIndex instance | +| `bruteforceIndex` | reference to the BruteForceIndex instance | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:57`_ + +### getCagraIndex + +```java +public CagraIndex getCagraIndex() +``` + +Gets the reference to the CAGRA index + +**Returns** + +an instance of CagraIndex + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:67`_ + +### getBruteforceIndex + +```java +public BruteForceIndex getBruteforceIndex() +``` + +Gets the reference to the Bruteforce index + +**Returns** + +an instance of BruteForceIndex + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:77`_ + +### getFieldName + +```java +public String getFieldName() +``` + +Gets the field name + +**Returns** + +field name + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:87`_ + +### getSegmentName + +```java +public String getSegmentName() +``` + +Gets the segment name + +**Returns** + +segment name + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:96`_ + +### getMaxDocs + +```java +public int getMaxDocs() +``` + +Gets the max docs + +**Returns** + +the max docs + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:105`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Closes this stream and releases any resources associated with it. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:121`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md new file mode 100644 index 0000000000..df1dbac445 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md @@ -0,0 +1,82 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery +--- + +# GPUKnnFloatVectorQuery + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class GPUKnnFloatVectorQuery extends KnnFloatVectorQuery +``` + +Extends `KnnFloatVectorQuery` for GPU-only search. + +When all index segments use `CuVS2510GPUVectorsReader`, `#rewrite` delegates a +single multi-partition search to cuVS, passing one Lucene segment per cuVS partition. cuVS +runs the per-partition CAGRA searches, applies distance post-processing, and performs the +cross-partition top-k merge internally; the returned arrays are mapped to Lucene doc IDs on +the host. The effective CAGRA algorithm (SINGLE_CTA or MULTI_KERNEL) is selected by cuVS +based on `searchAlgo` and `itopk_size`, with MULTI_KERNEL handling k beyond +SINGLE_CTA's per-partition cap. + +If the query has an explicit `filter`, or if any segment carries live-document deletes, +the acceptance mask (filter ∩ liveDocs) is packed into one `FilterBitsetHandle` per segment +and passed as that partition's filter. The host-side packed arrays are cached per unique +(filter, single-segment reader key, field) triple via `FilterBitsetCache`; the device +upload is cached inside the handle itself across threads. + +Falls back to the standard per-segment Lucene path when the optimized path cannot be +applied: mixed segment types, a missing CAGRA index for the field on any segment, or segments +whose built CAGRA graphs differ in degree (a single multi-partition request requires a uniform +graph degree, and a small segment can have its degree truncated at build time). + +## Public Members + +### GPUKnnFloatVectorQuery + +```java +public GPUKnnFloatVectorQuery( String field, float[] target, int k, Query filter, int iTopK, int searchWidth) +``` + +Initializes `GPUKnnFloatVectorQuery` with `CagraSearchParams.SearchAlgo#AUTO`, +and max_iterations auto-selected (0). + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | the vector field name | +| `target` | the query vector | +| `k` | the number of nearest neighbors to return | +| `filter` | optional pre-filter query | +| `iTopK` | CAGRA itopk_size parameter | +| `searchWidth` | CAGRA search_width parameter | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:90`_ + +### GPUKnnFloatVectorQuery + +```java +public GPUKnnFloatVectorQuery( String field, float[] target, int k, Query filter, int iTopK, int searchWidth, int threadBlockSize, int maxIterations, CagraSearchParams.SearchAlgo searchAlgo) +``` + +Initializes `GPUKnnFloatVectorQuery`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | the vector field name | +| `target` | the query vector | +| `k` | the number of nearest neighbors to return | +| `filter` | optional pre-filter query | +| `iTopK` | CAGRA itopk_size parameter | +| `searchWidth` | CAGRA search_width parameter | +| `threadBlockSize` | CAGRA thread_block_size (0 = auto) | +| `maxIterations` | CAGRA max_iterations (0 = auto) | +| `searchAlgo` | CAGRA search algorithm | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:108`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:72`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams.md new file mode 100644 index 0000000000..fba7ff55cf --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams.md @@ -0,0 +1,397 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams +--- + +# GPUSearchParams + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class GPUSearchParams +``` + +## Public Members + +### getWriterThreads + +```java +public int getWriterThreads() +``` + +Get the cuVS writer threads parameter + +**Returns** + +cuVS writer threads parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:118`_ + +### getIntermediateGraphDegree + +```java +public int getIntermediateGraphDegree() +``` + +Get the intermediate graph degree + +**Returns** + +the graph degree parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:127`_ + +### getGraphdegree + +```java +public int getGraphdegree() +``` + +Get the graph degree + +**Returns** + +the graph degree parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:136`_ + +### getCagraGraphBuildAlgo + +```java +public CagraGraphBuildAlgo getCagraGraphBuildAlgo() +``` + +Get the CAGRA build algorithm parameter value + +**Returns** + +the CAGRA build algorithm parameter value + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:145`_ + +### getIndexType + +```java +public IndexType getIndexType() +``` + +Get the index type parameter + +**Returns** + +the index type parameter + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:154`_ + +### getCuVSIvfPqParams + +```java +public CuVSIvfPqParams getCuVSIvfPqParams() +``` + +Get the instance of CuVSIvfPqParams + +**Returns** + +an instance of CuVSIvfPqParams + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:163`_ + +### getStrategy + +```java +public Strategy getStrategy() +``` + +Get the chosen strategy: + +When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set +When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + +**Returns** + +get the chosen `Strategy` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:176`_ + +### getCuvsDistanceType + +```java +public CuvsDistanceType getCuvsDistanceType() +``` + +Get the cuvs distance type + +**Returns** + +the distance type + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:185`_ + +### getnNDescentNumIterations + +```java +public int getnNDescentNumIterations() +``` + +get the number of Iterations to run if building with NN_DESCENT + +**Returns** + +the number of iterations for NN_DESCENT + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:194`_ + +### getBuildQuality + +```java +public int getBuildQuality() +``` + +Get the build quality handed to cuVS' build heuristic. Only consulted under the \{@link +Strategy#HEURISTIC\} strategy. + +**Returns** + +the build quality + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:204`_ + +### withWriterThreads + +```java +public Builder withWriterThreads(int writerThreads) +``` + +Set the number of cuVS writer threads while building the index +Valid range - Minimum: \{@value MIN_WRITER_THREADS\}, Maximum: \{@value MAX_WRITER_THREADS\} +Default value - \{@value DEFAULT_WRITER_THREADS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `writerThreads` | the number of cuVS writer threads | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:257`_ + +### withIntermediateGraphDegree + +```java +public Builder withIntermediateGraphDegree(int intermediateGraphDegree) +``` + +Set the intermediate graph degree to use while building CAGRA index +Valid range - Minimum: \{@value MIN_INT_GRAPH_DEG\}, Maximum: \{@value MAX_INT_GRAPH_DEG\} +Default value - \{@value DEFAULT_INT_GRAPH_DEGREE\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `intermediateGraphDegree` | the intermediate graph degree parameter | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:270`_ + +### withGraphDegree + +```java +public Builder withGraphDegree(int graphDegree) +``` + +Set the graph degree to use while building CAGRA index +Valid range - Minimum: \{@value MIN_GRAPH_DEG\}, Maximum: \{@value MAX_GRAPH_DEG\} +Default value - \{@value DEFAULT_GRAPH_DEGREE\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `graphDegree` | the graph degree parameter | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:283`_ + +### withCagraGraphBuildAlgo + +```java +public Builder withCagraGraphBuildAlgo(CagraGraphBuildAlgo cagraGraphBuildAlgo) +``` + +Set the CAGRA build algorithm. +Cannot be null, defaults to NN_DESCENT + +**Parameters** + +| Name | Description | +| --- | --- | +| `cagraGraphBuildAlgo` | the CAGRA build algorithm to use | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:295`_ + +### withIndexType + +```java +public Builder withIndexType(IndexType indexType) +``` + +Set the type of index to build - CAGRA, BRUTEFORCE, or both. +Cannot be null, defaults to CAGRA + +**Parameters** + +| Name | Description | +| --- | --- | +| `indexType` | the type of index to build | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:307`_ + +### withCuVSIvfPqParams + +```java +public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) +``` + +Set the instance of `CuVSIvfPqParams` + +**Parameters** + +| Name | Description | +| --- | --- | +| `cuVSIvfPqParams` | | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:318`_ + +### withStrategy + +```java +public Builder withStrategy(Strategy strategy) +``` + +Set the chosen strategy: + +When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set +When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + +Valid options - HEURISTIC, CUSTOM +Default value - HEURISTIC + +**Parameters** + +| Name | Description | +| --- | --- | +| `strategy` | , the strategy to choose | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:335`_ + +### withCuvsDistanceType + +```java +public Builder withCuvsDistanceType(CuvsDistanceType cuvsDistanceType) +``` + +Set the CuvsDistanceType + +**Parameters** + +| Name | Description | +| --- | --- | +| `cuvsDistanceType` | the CuvsDistanceType to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:346`_ + +### withNNDescentNumIterations + +```java +public Builder withNNDescentNumIterations(int nnDescentNumIterations) +``` + +Set the number of Iterations to run if building with NN_DESCENT + +Valid range - Minimum: \{@value MIN_NN_DESCENT_NUM_ITERATIONS\}, Maximum: \{@value MAX_NN_DESCENT_NUM_ITERATIONS\} +Default value - \{@value DEFAULT_NN_DESCENT_NUM_ITERATIONS\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `nnDescentNumIterations` | number of merge workers to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:360`_ + +### withBuildQuality + +```java +public Builder withBuildQuality(int buildQuality) +``` + +Set the build quality cuVS applies when deriving the build algorithm's parameters. Higher +values trade build cost for graph quality. + +Only consulted under the `Strategy#HEURISTIC` strategy. + +Valid range - Minimum: \{@value MIN_BUILD_QUALITY\}, unbounded above. cuVS documents any value +as valid, with values below 20 being the most practical. +Default value - \{@value DEFAULT_BUILD_QUALITY\} + +**Parameters** + +| Name | Description | +| --- | --- | +| `buildQuality` | the build quality to set | + +**Returns** + +instance of `Builder` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:378`_ + +### build + +```java +public GPUSearchParams build() +``` + +Creates and returns an instance of `GPUSearchParams` + +**Returns** + +instance of `GPUSearchParams` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:446`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java:15`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md new file mode 100644 index 0000000000..7f7dde5bbf --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md @@ -0,0 +1,104 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec +--- + +# Lucene101AcceleratedHNSWCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene101AcceleratedHNSWCodec extends FilterCodec +``` + +A codec that enables GPU-based accelerated HNSW capability and can be used +to accelerated indexing using GPUs and search using CPUs. Fallbacks to CPU +based indexing when used on a machine without a GPU and/or cuVS. + +## Public Members + +### Lucene101AcceleratedHNSWCodec + +```java +public Lucene101AcceleratedHNSWCodec() throws Exception +``` + +Default constructor for `Lucene101AcceleratedHNSWCodec`. + +**Throws** + +| Type | Description | +| --- | --- | +| `Exception` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:31`_ + +### Lucene101AcceleratedHNSWCodec + +```java +public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) +``` + +Constructor for `Lucene101AcceleratedHNSWCodec`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `name` | the codec's name | +| `delegate` | the delegate codec to filter | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:41`_ + +### Lucene101AcceleratedHNSWCodec + +```java +public Lucene101AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) throws Exception +``` + +Constructor for `Lucene101AcceleratedHNSWCodec`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | instance of `AcceleratedHNSWParams` | + +**Throws** + +| Type | Description | +| --- | --- | +| `Exception` | exception | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:52`_ + +### knnVectorsFormat + +```java +@Override public KnnVectorsFormat knnVectorsFormat() +``` + +Get the configured `KnnVectorsFormat`. + +**Returns** + +the instance of the `KnnVectorsFormat` + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:87`_ + +### setKnnFormat + +```java +public void setKnnFormat(KnnVectorsFormat format) +``` + +Set the `KnnVectorsFormat`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `format` | the `KnnVectorsFormat` to set | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:97`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java:21`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat.md new file mode 100644 index 0000000000..e84e99efc6 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat.md @@ -0,0 +1,81 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat +--- + +# Lucene99AcceleratedHNSWVectorsFormat + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat +``` + +cuVS based KnnVectorsFormat for indexing on GPU and searching on the CPU. + +## Public Members + +### Lucene99AcceleratedHNSWVectorsFormat + +```java +public Lucene99AcceleratedHNSWVectorsFormat() +``` + +Initializes `Lucene99AcceleratedHNSWVectorsFormat` with an instance +of `AcceleratedHNSWParams` with default parameter values. + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:56`_ + +### Lucene99AcceleratedHNSWVectorsFormat + +```java +public Lucene99AcceleratedHNSWVectorsFormat(AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Initializes `Lucene99AcceleratedHNSWVectorsFormat` with an instance +of `AcceleratedHNSWParams`. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:66`_ + +### fieldsWriter + +```java +@Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException +``` + +Returns a KnnVectorsWriter to write the vectors to the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:74`_ + +### fieldsReader + +```java +@Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException +``` + +Returns a KnnVectorsReader to read the vectors from the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:101`_ + +### getMaxDimensions + +```java +@Override public int getMaxDimensions(String fieldName) +``` + +Returns the maximum number of vector dimensions supported by this codec for the given field name. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:114`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java:27`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter.md new file mode 100644 index 0000000000..2f30caeb07 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter.md @@ -0,0 +1,102 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter +--- + +# Lucene99AcceleratedHNSWVectorsWriter + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter +``` + +This class extends upon the KnnVectorsWriter to +enable the creation of GPU-based accelerated HNSW based vector search. + +## Public Members + +### Lucene99AcceleratedHNSWVectorsWriter + +```java +public Lucene99AcceleratedHNSWVectorsWriter( SegmentWriteState state, AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException +``` + +Initializes `Lucene99AcceleratedHNSWVectorsWriter` + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the `org.apache.lucene.index.SegmentWriteState` | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | +| `flatVectorsWriter` | instance of the `org.apache.lucene.codecs.hnsw.FlatVectorsWriter` | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | IOException | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:86`_ + +### addField + +```java +@Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException +``` + +Add new field for indexing. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:128`_ + +### flush + +```java +@Override public void flush(int maxDoc, DocMap sortMap) throws IOException +``` + +Build the indexes and writes it to the disk. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:203`_ + +### mergeOneField + +```java +@Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException +``` + +Write field for merging. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:291`_ + +### finish + +```java +@Override public void finish() throws IOException +``` + +Called once at the end before close. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:300`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Closes the resources. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:320`_ + +### ramBytesUsed + +```java +@Override public long ramBytesUsed() +``` + +Returns the memory usage of this object in bytes. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:330`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java:52`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md new file mode 100644 index 0000000000..981e432792 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md @@ -0,0 +1,15 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec +--- + +# LuceneAcceleratedHNSWBinaryQuantizedCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWBinaryQuantizedCodec extends FilterCodec +``` + +CuVS based codec for GPU based vector search + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java:19`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat.md new file mode 100644 index 0000000000..b574aff9b1 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat.md @@ -0,0 +1,79 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorsformat +--- + +# LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat extends KnnVectorsFormat +``` + +cuVS based Binary Quantized KnnVectorsFormat for indexing on GPU and searching on the CPU. + +## Public Members + +### LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat + +```java +public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat() +``` + +Initializes `LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat` with default values. + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:52`_ + +### LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat + +```java +public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat( AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Initializes `LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat` with the given threads, graph degree, etc. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:61`_ + +### fieldsWriter + +```java +@Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException +``` + +Returns a KnnVectorsWriter to write the binary quantized vectors to the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:70`_ + +### fieldsReader + +```java +@Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException +``` + +Returns a KnnVectorsReader to read the binary quantized vectors from the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:99`_ + +### getMaxDimensions + +```java +@Override public int getMaxDimensions(String fieldName) +``` + +Returns the maximum number of vector dimensions supported by this codec for the given field name. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:112`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java:26`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter.md new file mode 100644 index 0000000000..91a4e67303 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter.md @@ -0,0 +1,102 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedvectorswriter +--- + +# LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter extends KnnVectorsWriter +``` + +This class extends upon the KnnVectorsWriter to enable the creation of GPU-based accelerated +vector search indexes. + +## Public Members + +### LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter + +```java +public LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter( SegmentWriteState state, AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException +``` + +Initializes `LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter` + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the `org.apache.lucene.index.SegmentWriteState` | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | +| `flatVectorsWriter` | instance of the `org.apache.lucene.codecs.hnsw.FlatVectorsWriter` | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | IOException | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:79`_ + +### addField + +```java +@Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException +``` + +Add new field for indexing. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:127`_ + +### flush + +```java +@Override public void flush(int maxDoc, DocMap sortMap) throws IOException +``` + +Build the indexes and writes it to the disk. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:215`_ + +### mergeOneField + +```java +@Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException +``` + +Write field for merging. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:302`_ + +### finish + +```java +@Override public void finish() throws IOException +``` + +Called once at the end before close. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:333`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Closes the resources. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:353`_ + +### ramBytesUsed + +```java +@Override public long ramBytesUsed() +``` + +Returns the memory usage of this object in bytes. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:362`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java:57`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md new file mode 100644 index 0000000000..e31e66c7fc --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md @@ -0,0 +1,15 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec +--- + +# LuceneAcceleratedHNSWScalarQuantizedCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWScalarQuantizedCodec extends FilterCodec +``` + +CuVS based codec for GPU based vector search + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java:19`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat.md new file mode 100644 index 0000000000..1d249bc2df --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat.md @@ -0,0 +1,79 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorsformat +--- + +# LuceneAcceleratedHNSWScalarQuantizedVectorsFormat + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWScalarQuantizedVectorsFormat extends KnnVectorsFormat +``` + +cuVS based Scalar Quantized KnnVectorsFormat for indexing on GPU and searching on the CPU. + +## Public Members + +### LuceneAcceleratedHNSWScalarQuantizedVectorsFormat + +```java +public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat() +``` + +Initializes `LuceneAcceleratedHNSWScalarQuantizedVectorsFormat` with default values. + +**Throws** + +| Type | Description | +| --- | --- | +| `LibraryException` | if the native library fails to load | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:47`_ + +### LuceneAcceleratedHNSWScalarQuantizedVectorsFormat + +```java +public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat( AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Initializes `LuceneAcceleratedHNSWScalarQuantizedVectorsFormat` with the given threads, graph degree, etc. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:56`_ + +### fieldsWriter + +```java +@Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException +``` + +Returns a KnnVectorsWriter to write the scalar quantized vectors to the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:65`_ + +### fieldsReader + +```java +@Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException +``` + +Returns a KnnVectorsReader to read the scalar quantized vectors from the index. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:91`_ + +### getMaxDimensions + +```java +@Override public int getMaxDimensions(String fieldName) +``` + +Returns the maximum number of vector dimensions supported by this Codec for the given field name. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:104`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java:24`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter.md new file mode 100644 index 0000000000..9c51cb26e9 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter.md @@ -0,0 +1,102 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedvectorswriter +--- + +# LuceneAcceleratedHNSWScalarQuantizedVectorsWriter + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneAcceleratedHNSWScalarQuantizedVectorsWriter extends KnnVectorsWriter +``` + +This class extends upon the KnnVectorsWriter to enable the creation of GPU-based accelerated +vector search indexes. + +## Public Members + +### LuceneAcceleratedHNSWScalarQuantizedVectorsWriter + +```java +public LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( SegmentWriteState state, AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException +``` + +Initializes `LuceneAcceleratedHNSWScalarQuantizedVectorsWriter` + +**Parameters** + +| Name | Description | +| --- | --- | +| `state` | instance of the `org.apache.lucene.index.SegmentWriteState` | +| `acceleratedHNSWParams` | An instance of `AcceleratedHNSWParams` | +| `flatVectorsWriter` | instance of the `org.apache.lucene.codecs.hnsw.FlatVectorsWriter` | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | IOException | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:89`_ + +### addField + +```java +@Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException +``` + +Add new field for indexing. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:136`_ + +### flush + +```java +@Override public void flush(int maxDoc, DocMap sortMap) throws IOException +``` + +Build the indexes and writes it to the disk. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:241`_ + +### mergeOneField + +```java +@Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException +``` + +Write field for merging. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:326`_ + +### finish + +```java +@Override public void finish() throws IOException +``` + +Called once at the end before close. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:357`_ + +### close + +```java +@Override public void close() throws IOException +``` + +Closes the resources. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:377`_ + +### ramBytesUsed + +```java +@Override public long ramBytesUsed() +``` + +Returns the memory usage of this object in bytes. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:386`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java:56`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneprovider.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneprovider.md new file mode 100644 index 0000000000..e46ee10813 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneprovider.md @@ -0,0 +1,15 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneprovider +--- + +# LuceneProvider + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class LuceneProvider +``` + +Dynamically loads Lucene format, reader, and writer classes with a fallback mechanism. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java:31`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider.md new file mode 100644 index 0000000000..f416e55816 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider.md @@ -0,0 +1,99 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider +--- + +# ThreadLocalCuVSResourcesProvider + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class ThreadLocalCuVSResourcesProvider +``` + +Provides a mechanism to create ThreadLocal based CuVSResource instances. + +## Public Members + +### getCuVSResourcesInstance + +```java +public static CuVSResources getCuVSResourcesInstance() +``` + +Gets an instance of CuVSResources for the accessing thread. + +**Returns** + +an instance of CuVSResources + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:30`_ + +### setCuVSResourcesInstance + +```java +public static void setCuVSResourcesInstance(CuVSResources resources) +``` + +Sets the instance of CuVSResources + +**Parameters** + +| Name | Description | +| --- | --- | +| `resources` | the instance of CuVSResources to set | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:39`_ + +### resolveWorkspacePoolBytes + +```java +static long resolveWorkspacePoolBytes(String raw) +``` + +Resolves a raw workspace-pool property value to a 256-byte-aligned size. Zero or an absent +value disables the per-resources pool. Invalid, negative, or unalignable values warn and also +disable it. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:80`_ + +### closeCuVSResourcesInstance + +```java +public static void closeCuVSResourcesInstance() +``` + +Attempts to close the thread's `CuVSResources` instance. + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:122`_ + +### assertIsSupported + +```java +public static void assertIsSupported() throws UnsupportedOperationException +``` + +Checks if cuVS is supported and throws `UnsupportedOperationException` otherwise. + +**Throws** + +| Type | Description | +| --- | --- | +| `UnsupportedOperationException` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:135`_ + +### isSupported + +```java +public static boolean isSupported() +``` + +Checks if cuVS is supported. + +**Returns** + +true if cuVS is supported else false + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:146`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java:16`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-utils.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-utils.md new file mode 100644 index 0000000000..0d2a9cb1a6 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-utils.md @@ -0,0 +1,223 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-utils +--- + +# Utils + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Utils +``` + +This class provides common static utility methods. + +## Public Members + +### handleThrowable + +```java +static RuntimeException handleThrowable(Throwable t) throws IOException +``` + +A utility method that rethrows known throwable types without changing their identity. + +In particular, `Error` instances must not be converted to a \{@link +RuntimeException\}; callers rely on errors retaining their original type and stack trace. + +This method never returns normally; its return type exists solely so callers can write +`throw handleThrowable(t);`, letting the compiler verify that the enclosing statement +always completes abruptly. + +**Parameters** + +| Name | Description | +| --- | --- | +| `t` | the throwable object | + +**Returns** + +never returns; always throws + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:43`_ + +### createFloatMatrix + +```java +static CuVSMatrix createFloatMatrix(List data, int dimensions, CuVSResources resources) +``` + +A method to build a CuVSMatrix from a list of float vectors. + +Uses CuVSMatrix.Builder to copy vectors directly to device memory +without creating intermediate heap arrays. + +**Parameters** + +| Name | Description | +| --- | --- | +| `data` | The float vectors | +| `dimensions` | The number float elements in each vector | +| `resources` | The CuVS resources for device matrix creation | + +**Returns** + +an instance of CuVSMatrix + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:63`_ + +### createByteMatrix + +```java +static CuVSMatrix createByteMatrix( List data, int bytesPerVector, CuVSResources resources) +``` + +A method to build a CuVSMatrix from a list of byte vectors (for binary quantized vectors). + +Uses CuVSMatrix.Builder to copy vectors directly to device memory +without creating intermediate heap arrays. + +**Parameters** + +| Name | Description | +| --- | --- | +| `data` | The byte vectors (packed bits for binary quantization) | +| `bytesPerVector` | The number of bytes in each vector | +| `resources` | The CuVS resources for device matrix creation | + +**Returns** + +an instance of CuVSMatrix with BYTE data type + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:92`_ + +### createByteMatrixFromArray + +```java +static CuVSMatrix createByteMatrixFromArray( byte[][] data, int bytesPerVector, CuVSResources resources) +``` + +A method to build a CuVSMatrix from a 2D byte array (for binary quantized vectors). + +**Parameters** + +| Name | Description | +| --- | --- | +| `data` | The 2D byte array (packed bits for binary quantization) | +| `bytesPerVector` | The number of bytes in each vector | +| `resources` | The CuVS resources for device matrix creation | + +**Returns** + +an instance of CuVSMatrix with BYTE data type + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:119`_ + +### nanosToMillis + +```java +static long nanosToMillis(long nanos) +``` + +A utility method to convert nanoseconds to milliseconds. + +**Parameters** + +| Name | Description | +| --- | --- | +| `nanos` | | + +**Returns** + +milliseconds + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:141`_ + +### cuVSResourcesOrNull + +```java +static CuVSResources cuVSResourcesOrNull() +``` + +Creates an instance of CuVSResources. + +**Returns** + +an instance of CuVSResources + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:150`_ + +### handleThrowableWithIgnore + +```java +static void handleThrowableWithIgnore(Throwable t, String msg) throws IOException +``` + +A utility method that conditionally ignores certain throwable objects + +**Parameters** + +| Name | Description | +| --- | --- | +| `t` | the throwable object | +| `msg` | the message to check | + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:178`_ + +### createListFromMergedVectors + +```java +static List createListFromMergedVectors(FloatVectorValues mergedVectorValues) throws IOException +``` + +Creates a list of float vectors from the input + +**Parameters** + +| Name | Description | +| --- | --- | +| `mergedVectorValues` | instance of `FloatVectorValues` | + +**Returns** + +a list of float arrays + +**Throws** + +| Type | Description | +| --- | --- | +| `IOException` | I/O Exception | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:192`_ + +### info + +```java +static void info(InfoStream infoStream, String component, String msg) +``` + +Utility to print info/debug messages via InfoStream. + +**Parameters** + +| Name | Description | +| --- | --- | +| `infoStream` | the writer's infostream | +| `component` | the name of the index writer | +| `msg` | the log message to push via the InfoStream | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:210`_ + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java:26`_ diff --git a/fern/scripts/generate_api_reference.py b/fern/scripts/generate_api_reference.py index 9a02eb1d0d..9acfa9f0f6 100755 --- a/fern/scripts/generate_api_reference.py +++ b/fern/scripts/generate_api_reference.py @@ -19,7 +19,7 @@ import shutil import textwrap from pathlib import Path -from typing import Iterable +from typing import Callable, Iterable REPO_DIR = Path(__file__).resolve().parents[2] @@ -32,6 +32,9 @@ REPO_DIR / "java" / "cuvs-java" / "src" / "main" / "java", REPO_DIR / "java" / "cuvs-java" / "src" / "main" / "java22", ] +LUCENE_SOURCE_DIRS = [ + REPO_DIR / "java" / "cuvs-lucene" / "src" / "main" / "java", +] API_NAV_SECTIONS = [ ("C API Documentation", "c_api", "c-api-documentation", "C API", "c-api"), ( @@ -55,6 +58,13 @@ "Java API", "java-api", ), + ( + "Lucene API Documentation", + "lucene_api", + "lucene-api-documentation", + "Lucene API", + "lucene-api", + ), ( "Rust API Documentation", "rust_api", @@ -87,6 +97,7 @@ "cpp_api", "python_api", "java_api", + "lucene_api", "rust_api", "go_api", ] @@ -119,6 +130,21 @@ ), ] API_DECORATOR_LEAK_RE = re.compile(r"\bCUVS_EXPORT\b") +# ``cuvs-lucene`` classes all live in one package, so the Lucene type each +# one extends is the only structural signal available for grouping. These are +# Lucene API names, which are stable across cuVS releases. +LUCENE_EXTENSION_POINTS = frozenset( + { + "FilterCodec", + "HnswGraph", + "KnnFieldVectorsWriter", + "KnnFloatVectorQuery", + "KnnVectorsFormat", + "KnnVectorsReader", + "KnnVectorsWriter", + } +) +JAVA_SUPERTYPE_RE = re.compile(r"\b(?:extends|implements)\s+(?P\w+)") PUBLIC_JAVA_TYPE_RE = re.compile( r"\bpublic\s+(?:abstract\s+|final\s+|sealed\s+|non-sealed\s+)?" r"(?Pclass|interface|enum|record)\s+(?P[A-Za-z_]\w*)" @@ -449,6 +475,7 @@ def main(argv: list[str] | None = None) -> int: ) generate_python_api_pages() generate_java_api_pages() + generate_lucene_api_pages() generate_rust_api_pages() generate_go_api_pages() update_api_navigation() @@ -462,6 +489,7 @@ def remove_old_api_pages() -> None: FERN_PAGES / "cpp_api", FERN_PAGES / "python_api", FERN_PAGES / "java_api", + FERN_PAGES / "lucene_api", FERN_PAGES / "rust_api", FERN_PAGES / "go_api", ]: @@ -2965,28 +2993,61 @@ def render_python_symbol(symbol: PythonSymbol) -> list[str]: def generate_java_api_pages() -> None: - out_dir = FERN_PAGES / "java_api" + generate_jvm_api_pages( + directory="java_api", + title="Java API Documentation", + intro=( + "These pages are generated from the Java source files in " + "`java/cuvs-java/src/main`." + ), + source_dirs=JAVA_SOURCE_DIRS, + group_of=java_api_group, + ) + + +def generate_lucene_api_pages() -> None: + generate_jvm_api_pages( + directory="lucene_api", + title="Lucene API Documentation", + intro=( + "These pages are generated from the Java source files in " + "`java/cuvs-lucene/src/main`." + ), + source_dirs=LUCENE_SOURCE_DIRS, + group_of=lucene_api_group, + ) + + +def generate_jvm_api_pages( + *, + directory: str, + title: str, + intro: str, + source_dirs: list[Path], + group_of: Callable[[JavaClass], str], +) -> None: + out_dir = FERN_PAGES / directory out_dir.mkdir(parents=True, exist_ok=True) - classes = collect_java_classes() + classes = collect_java_classes(source_dirs) index_lines = [ - "# Java API Documentation", + f"# {title}", "", - "These pages are generated from the Java source files in `java/cuvs-java/src/main`.", + intro, "", ] grouped: dict[str, list[tuple[str, str]]] = defaultdict(list) for klass in classes: - grouped[java_api_group(klass)].append( - (klass.name, api_doc_url("java_api", java_slug(klass))) + grouped[group_of(klass)].append( + (klass.name, api_doc_url(directory, java_slug(klass))) ) append_api_index_groups(index_lines, grouped) write_page(out_dir / "index.md", index_lines) for klass in classes: - API_NAV_TITLE_OVERRIDES[("java_api", java_slug(klass))] = klass.name + API_NAV_TITLE_OVERRIDES[(directory, java_slug(klass))] = klass.name lines = [ - *api_frontmatter(api_page_route("java_api", java_slug(klass))), + *api_frontmatter(api_page_route(directory, java_slug(klass))), f"# {klass.name}", "", f"_Java package: `{klass.package}`_", @@ -3022,7 +3083,7 @@ def generate_java_api_pages() -> None: lines.extend([f"_Source: `{klass.source}:{member.line}`_", ""]) lines.extend([f"_Source: `{klass.source}:{klass.line}`_", ""]) write_page( - out_dir / f"{api_page_route('java_api', java_slug(klass))}.md", + out_dir / f"{api_page_route(directory, java_slug(klass))}.md", lines, ) @@ -3045,9 +3106,21 @@ def java_api_group(klass: JavaClass) -> str: return "Common" -def collect_java_classes() -> list[JavaClass]: +def lucene_api_group(klass: JavaClass) -> str: + """Group a Lucene class by whether it implements a Lucene vector API. + + The supertype is already part of ``klass.signature``, so a new codec + generation classifies itself without touching this function. + """ + match = JAVA_SUPERTYPE_RE.search(klass.signature) + if match is not None and match.group("name") in LUCENE_EXTENSION_POINTS: + return "Codecs and Formats" + return "Common" + + +def collect_java_classes(roots: list[Path]) -> list[JavaClass]: classes: list[JavaClass] = [] - for root in JAVA_SOURCE_DIRS: + for root in roots: for path in sorted(root.rglob("*.java")): if ( "internal" in path.relative_to(root).parts