diff --git a/docs/clean-full-website.sh b/docs/clean-full-website.sh index 4b89917597d..9ef2f44ac30 100755 --- a/docs/clean-full-website.sh +++ b/docs/clean-full-website.sh @@ -39,23 +39,10 @@ EOF cat > "$docs_src/integrations/.pages" <<'EOF' nav: - Overview: index.md - - Database: - - LanceDB: https://docs.lancedb.com/ - - Language SDKs: - - Python (PyLance): https://lance-format.github.io/lance-python-doc - - Rust (docs.rs): https://docs.rs/lance/latest/lance - - Java (Javadoc): https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html - - Query & Compute Engines: - - Apache Spark: spark - - DuckDB: duckdb - - Trino: trino - - Apache DataFusion: datafusion.md - - Ray: ray - - PostgreSQL: https://github.com/lance-format/pglance - - ML & AI Ecosystem: - - PyTorch: pytorch.md - - Hugging Face: huggingface - - TensorFlow: tensorflow.md + - Apache DataFusion: datafusion.md + - PostgreSQL: https://github.com/lance-format/pglance + - PyTorch: pytorch.md + - TensorFlow: tensorflow.md EOF mkdir -p "$docs_src/format/catalog/dir" diff --git a/docs/make-full-website.sh b/docs/make-full-website.sh index b751e7d9677..1cd8df05847 100755 --- a/docs/make-full-website.sh +++ b/docs/make-full-website.sh @@ -227,14 +227,21 @@ else warn_missing_repo "Lance Namespace Impls docs" "$namespace_impls_repo/docs/src" fi +integration_entries=( +) + +while IFS= read -r line; do + integration_entries+=("$line") +done < <(sed -n '2,$p' "$docs_src/integrations/.pages") + if copy_duckdb_docs "$duckdb_repo/docs" "$docs_src/integrations/duckdb"; then - : + integration_entries+=(" - DuckDB: duckdb") else warn_missing_repo "Lance DuckDB docs" "$duckdb_repo/docs" fi if copy_docs_dir "$huggingface_repo/docs/src" "$docs_src/integrations/huggingface"; then - : + integration_entries+=(" - Huggingface: huggingface") else warn_missing_repo "Lance HuggingFace docs" "$huggingface_repo/docs/src" fi @@ -249,66 +256,41 @@ if path.exists(): text = path.read_text() path.write_text(text.replace('https://lance.org/format/table/index/scalar/fts/#tokenizers', 'https://lance.org/format/index/scalar/fts/#tokenizers')) PY + integration_entries+=(" - Apache Spark: spark") else warn_missing_repo "Lance Spark docs" "$spark_repo/docs/src" fi if copy_docs_dir "$ray_repo/docs/src" "$docs_src/integrations/ray"; then - : + integration_entries+=(" - Ray: ray") else warn_missing_repo "Lance Ray docs" "$ray_repo/docs/src" fi if copy_docs_dir "$trino_repo/docs/src" "$docs_src/integrations/trino"; then - : + integration_entries+=(" - Trino: trino") else warn_missing_repo "Lance Trino docs" "$trino_repo/docs/src" fi -has_context_docs=false if copy_docs_dir "$context_repo/docs/src" "$docs_src/integrations/context"; then - has_context_docs=true + integration_entries+=(" - Lance Context: context") else warn_missing_repo "Lance Context docs" "$context_repo/docs/src" fi -has_flink_docs=false if copy_docs_dir "$flink_repo/docs/src" "$docs_src/integrations/flink"; then - has_flink_docs=true + integration_entries+=(" - Apache Flink: flink") else warn_missing_repo "Lance Flink docs" "$flink_repo/docs/src" fi { - cat <<'EOF' -nav: - - Overview: index.md - - Database: - - LanceDB: https://docs.lancedb.com/ - - Language SDKs: - - Python (PyLance): https://lance-format.github.io/lance-python-doc - - Rust (docs.rs): https://docs.rs/lance/latest/lance - - Java (Javadoc): https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html - - Query & Compute Engines: - - Apache Spark: spark - - DuckDB: duckdb - - Trino: trino - - Apache DataFusion: datafusion.md - - Ray: ray -EOF - if [ "$has_flink_docs" = true ]; then - echo " - Apache Flink: flink" - fi - cat <<'EOF' - - PostgreSQL: https://github.com/lance-format/pglance - - ML & AI Ecosystem: - - PyTorch: pytorch.md - - Hugging Face: huggingface - - TensorFlow: tensorflow.md -EOF - if [ "$has_context_docs" = true ]; then - echo " - Lance Context: context" - fi + echo "nav:" + for entry in "${integration_entries[@]}"; do + [ -n "$entry" ] || continue + echo "$entry" + done } > "$docs_src/integrations/.pages" mkdir -p "$docs_src/community/project-specific/lance" diff --git a/docs/snippets/quickstart/lancedb_quickstart.py b/docs/snippets/quickstart/lancedb_quickstart.py index 8179786e064..e4c92027158 100644 --- a/docs/snippets/quickstart/lancedb_quickstart.py +++ b/docs/snippets/quickstart/lancedb_quickstart.py @@ -1,4 +1,4 @@ -# Runnable source for docs/src/quickstart/lancedb.md. The sections between the +# Runnable source for docs/src/quickstart/index.md. The sections between the # `--8<--` markers are embedded in the page, and CI checks what each one # displays against outputs/ (see tests/snippet_runner.py). # Sections import what they use and end with an expression to display, like diff --git a/docs/src/.pages b/docs/src/.pages index 5c370f90420..e0e65f974ee 100644 --- a/docs/src/.pages +++ b/docs/src/.pages @@ -6,3 +6,4 @@ nav: - Examples: examples - Integrations: integrations - Community: community + - SDK Docs: sdk_docs.md \ No newline at end of file diff --git a/docs/src/integrations/.pages b/docs/src/integrations/.pages index 5f30a5c5356..ab9c2c0dcee 100644 --- a/docs/src/integrations/.pages +++ b/docs/src/integrations/.pages @@ -1,19 +1,6 @@ nav: - Overview: index.md - - Database: - - LanceDB: https://docs.lancedb.com/ - - Language SDKs: - - Python (PyLance): https://lance-format.github.io/lance-python-doc - - Rust (docs.rs): https://docs.rs/lance/latest/lance - - Java (Javadoc): https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html - - Query & Compute Engines: - - Apache Spark: spark - - DuckDB: duckdb - - Trino: trino - - Apache DataFusion: datafusion.md - - Ray: ray - - PostgreSQL: https://github.com/lance-format/pglance - - ML & AI Ecosystem: - - PyTorch: pytorch.md - - Hugging Face: huggingface - - TensorFlow: tensorflow.md + - Apache DataFusion: datafusion.md + - PostgreSQL: https://github.com/lance-format/pglance + - PyTorch: pytorch.md + - TensorFlow: tensorflow.md diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md index 1867dfa79a7..9b047b873d7 100644 --- a/docs/src/integrations/index.md +++ b/docs/src/integrations/index.md @@ -9,28 +9,19 @@ Because Lance is built on Apache Arrow and exposes a stable Rust core with Python and Java bindings, it can be consumed from query engines, ML frameworks, databases, and catalog systems without copying or converting data. -This page lists the database, SDKs, engines, and ecosystem integrations maintained by the Lance community. Some live in this repository as first-party +This page lists the integrations maintained by the Lance community. Some live in this repository as first-party Rust, Python, and Java APIs; others are maintained as dedicated subprojects under the [lance-format](https://github.com/lance-format) GitHub organization. -## Database and Language SDKs - -| Layer | Project | Documentation | -|---|---|---| -| **Database** | **LanceDB** — Embedded and distributed AI retrieval database built on Lance | [Quickstart](../quickstart/lancedb.md) · [LanceDB Docs ↗](https://docs.lancedb.com/) | -| **Language SDK** | **Python (`pylance`)** — Native Python SDK for Lance datasets and fragments | [Quickstart](../quickstart/pylance.md) · [Python API Docs ↗](https://lance-format.github.io/lance-python-doc) | -| **Language SDK** | **Rust (`lance`)** — Native Rust core and dataset API | [Quickstart](../quickstart/rust.md) · [docs.rs/lance ↗](https://docs.rs/lance/latest/lance) | -| **Language SDK** | **Java (`org.lance:lance-core`)** — JNI and Arrow C Data Interface bindings for the JVM | [Quickstart](../quickstart/java.md) · [Javadoc ↗](https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html) | - ## Query Engines and SQL | Integration | Description | Source | |---|---|---| +| [Apache DataFusion](datafusion.md) | Register Lance datasets as DataFusion tables and run SQL with predicate/projection pushdown. Available in both Rust and Python. | Built-in | | [Apache Spark](spark/index.md) | Read and write Lance datasets from Spark SQL and DataFrames. | [lance-format/lance-spark](https://github.com/lance-format/lance-spark) | +| [Apache Doris](https://doris.apache.org/docs/4.x/lakehouse/catalogs/lance-catalog/) | Query Lance datasets with Doris SQL through filesystem and REST catalogs, with predicate pushdown and vector search. Read-only. | [apache/doris](https://github.com/apache/doris) | | [DuckDB](duckdb/index.md) | Query Lance datasets directly from DuckDB. | [lance-format/lance-duckdb](https://github.com/lance-format/lance-duckdb) | | [Trino](trino/index.md) | Federate Lance into Trino alongside other connectors. | [lance-format/lance-trino](https://github.com/lance-format/lance-trino) | -| [Apache DataFusion](datafusion.md) | Register Lance datasets as DataFusion tables and run SQL with predicate/projection pushdown. Available in both Rust and Python. | Built-in | -| [Apache Doris](https://doris.apache.org/docs/4.x/lakehouse/catalogs/lance-catalog/) | Query Lance datasets with Doris SQL through filesystem and REST catalogs, with predicate pushdown and vector search. Read-only. | [apache/doris](https://github.com/apache/doris) | ## Machine Learning and AI diff --git a/docs/src/quickstart/.pages b/docs/src/quickstart/.pages index e78c26458f8..a1a098466c8 100644 --- a/docs/src/quickstart/.pages +++ b/docs/src/quickstart/.pages @@ -1,14 +1,10 @@ nav: - - Overview: index.md - - Database: - - LanceDB: lancedb.md - - Language SDKs: - - Python (PyLance): - - Getting Started: pylance.md - - Versioning: versioning.md - - Vector Search: vector-search.md - - Full-Text Search: full-text-search.md - - Rust: rust.md - - Java: java.md - - Query & Compute Engines: - - Apache Spark: spark.md + - LanceDB: index.md + - Apache Spark: spark.md + - PyLance: + - Getting Started: pylance.md + - Versioning: versioning.md + - Vector Search: vector-search.md + - Full-Text Search: full-text-search.md + - Rust: rust.md + - Java: java.md diff --git a/docs/src/quickstart/index.md b/docs/src/quickstart/index.md index d5e4b2ba9ff..76db5e51cdf 100644 --- a/docs/src/quickstart/index.md +++ b/docs/src/quickstart/index.md @@ -1,56 +1,144 @@ --- -title: Getting Started with Lance -description: Choose the right entry point in the Lance stack — LanceDB, Python/Rust/Java SDKs, or SQL and compute engines +title: LanceDB Quickstart +description: Get started with Lance using LanceDB - create a table, query it, and run vector and full-text search --- -# Getting Started with Lance +# Getting Started with LanceDB -Lance is an open lakehouse and columnar table format for multimodal AI. Every tool in the Lance ecosystem reads and writes the same on-disk format, so you can start at the layer that matches your application and interoperate across layers without copying or converting data. +Lance is an open table format. You can work with Lance tables from many engines, and this quickstart covers: -## The Lance Stack +- **LanceDB** (this page) is a retrieval database built on Lance. It runs embedded in your application or as a distributed service. +- **[Apache Spark](spark.md)** runs SQL and distributed data processing on Lance tables. +- **Format SDKs ([PyLance](pylance.md), [Rust](rust.md), [Java](java.md))** provide low-level control over Lance datasets when building custom engines or pipelines. -```text -+---------------------------------------------------------------------+ -| 1. Database LanceDB (Embedded & Distributed) | -+---------------------------------------------------------------------+ -| 2. Language SDKs Python (PyLance), Rust, Java | -+---------------------------------------------------------------------+ -| 3. Query & Compute Engines Spark, DuckDB, Trino, DataFusion, Ray | -+=====================================================================+ -| Lance Open Format & Catalog Spec | -| (Columnar Files, ACID Versioning, Blob v2, Indexes) | -+---------------------------------------------------------------------+ +All of these tools read and write the same format, so a table you create with one of them can be opened with the others. + +In this guide, you'll create a small product catalog with LanceDB, filter it, search it by vector similarity and by keyword, modify it, and look back at earlier versions. + +## Install LanceDB + +This guide displays results as pandas DataFrames, so install pandas as well: + +```bash +--8<-- "snippets/quickstart/lancedb-install.sh" +``` + +## Connect to a Database + +This guide uses LanceDB in embedded mode, where a database is a directory of Lance tables, on local disk or in object storage such as `s3://`. Connect to a local path; LanceDB creates it if it doesn't exist: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:connect" +``` + +The same client connects to a distributed LanceDB deployment when you pass a `db://` URI instead of a path. + +## Create a Table + +Create a table from a list of dictionaries. LanceDB infers the schema and stores the `vector` column as a fixed-size vector: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:create" ``` -## Choose Your Entry Point +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.create.txt" +``` + +LanceDB also accepts pandas DataFrames, Polars DataFrames, and Arrow tables as input. -### 1. Database — LanceDB +## Filter Rows -Choose **LanceDB** when you are building an AI application, RAG pipeline, or search service and want table management, vector search, BM25 full-text search, and hybrid reranking out of the box. +Use SQL expressions to filter rows and choose columns: -- **[LanceDB Quickstart](lancedb.md)** — Create a table, run vector and keyword search, update rows, and time travel in 5 minutes. -- **[LanceDB Documentation ↗](https://docs.lancedb.com/)** — Embeddings, hybrid reranking, TypeScript/Python/Rust database clients, and Cloud/Enterprise deployments. +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:filter" +``` -### 2. Language SDKs — Python, Rust, and Java +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.filter.txt" +``` -Choose a **Language SDK** when you want direct, low-level control over Lance datasets, fragments, custom Arrow schemas, multimodal [Blob v2 columns](../guide/blob.md), or [distributed write transactions](../guide/distributed_write.md). +## Search by Vector Similarity + +Pass a query vector to `search` to find the nearest rows. The `_distance` column holds each row's distance from the query: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:vector-search" +``` -| Language | Package | Quickstart | API Reference | -|---|---|---|---| -| **Python** | `pylance` | **[Python (PyLance) Quickstart](pylance.md)** · [Versioning](versioning.md) · [Vector Search](vector-search.md) · [Full-Text Search](full-text-search.md) | [Python API Docs ↗](https://lance-format.github.io/lance-python-doc) | -| **Rust** | `lance` | **[Rust Quickstart](rust.md)** | [docs.rs/lance ↗](https://docs.rs/lance/latest/lance) | -| **Java** | `org.lance:lance-core` | **[Java Quickstart](java.md)** | [Javadoc ↗](https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html) | +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.vector-search.txt" +``` + +You can combine vector search with a filter: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:filtered-vector-search" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.filtered-vector-search.txt" +``` -### 3. Query & Compute Engines +On a small table like this one, LanceDB compares the query against every vector. For larger tables, create a [vector index](https://docs.lancedb.com/indexing) to keep searches fast. -Choose a **Query or Compute Engine** when you want to run distributed SQL, lakehouse ETL, analytics, or large-scale ML training over Lance tables alongside existing data pipelines. +## Search by Keyword -| Engine | Typical Workload | Get Started | -|---|---|---| -| **Apache Spark** | Distributed SQL, DDL/DML, and batch ETL with Spark SQL and PySpark | **[Spark Quickstart](spark.md)** · [Spark Connector Docs](../integrations/spark/index.md) | -| **DuckDB / Trino / DataFusion** | Interactive SQL analytics and federated queries over Lance datasets | [DataFusion](../integrations/datafusion.md) · [All SQL Integrations](../integrations/index.md#query-engines-and-sql) | -| **Ray / PyTorch / Flink** | Distributed data processing, streaming ingestion, and ML training loaders | [PyTorch](../integrations/pytorch.md) · [All ML & Stream Integrations](../integrations/index.md#machine-learning-and-ai) | +Create a full-text search index on a text column, then search it with keywords. The `_score` column holds each row's relevance score: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:fts" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.fts.txt" +``` + +## Add, Update, and Delete Rows + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:modify" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.modify.txt" +``` + +## Time Travel + +Every change creates a new version of the table: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:version" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.version.txt" +``` + +Earlier versions stay readable. Check out version 1 to see the table as it was when you created it: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:checkout" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.checkout.txt" +``` + +Return to the latest version to continue making changes: + +```python +--8<-- "snippets/quickstart/lancedb_quickstart.py:checkout-latest" +``` + +```output +--8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.checkout-latest.txt" +``` -## One Format Across Every Layer +## Next Steps -Because all layers share the [Lance format specification](../format/index.md), you can mix tools freely in a single pipeline — for example, ingest and backfill features at scale with **Apache Spark**, build vector and full-text indexes with the **Python or Rust SDK**, and serve low-latency hybrid queries from **LanceDB**. +- Read the [LanceDB documentation](https://docs.lancedb.com/) for embeddings, hybrid search, reranking, distributed deployments, and the TypeScript and Rust SDKs. +- See the [Spark quickstart](spark.md) to process Lance tables at scale with SQL. +- See the format SDK quickstarts for [PyLance](pylance.md), [Rust](rust.md), and [Java](java.md) to work directly with the Lance format. diff --git a/docs/src/quickstart/java.md b/docs/src/quickstart/java.md index b7dc37f478b..aef5c970311 100644 --- a/docs/src/quickstart/java.md +++ b/docs/src/quickstart/java.md @@ -5,7 +5,7 @@ description: Get started with the Lance Java SDK — write Arrow batches, scan w # Getting Started with Java -The `org.lance:lance-core` artifact provides Java bindings to the Lance Rust core via JNI and Apache Arrow's C Data Interface. It powers JVM compute connectors (such as [Apache Spark](spark.md), Trino, and Flink) and can be embedded directly in Java and Scala services. +The `org.lance:lance-core` artifact is the low-level Java SDK for the Lance format, providing JNI and Apache Arrow C Data Interface bindings to the Rust core. If you are running analytics or data pipelines on the JVM, prefer a query engine such as [Apache Spark](spark.md) or [Trino](../integrations/trino/index.md); use `lance-core` directly when building custom engine connectors or storage integrations. ## Add Dependencies diff --git a/docs/src/quickstart/lancedb.md b/docs/src/quickstart/lancedb.md deleted file mode 100644 index 4a7d03f21d5..00000000000 --- a/docs/src/quickstart/lancedb.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: LanceDB Quickstart -description: Get started with Lance using LanceDB - create a table, query it, and run vector and full-text search ---- - -# Getting Started with LanceDB - -**LanceDB** is an open-source retrieval database built on the Lance format. It runs embedded in your application (local disk or object storage such as `s3://`) or as a distributed service (`db://`), handling table management, indexing, and search out of the box. - -If you prefer to work directly with low-level Lance datasets or distributed SQL engines, see the [Getting Started Overview](index.md). - -In this guide, you'll create a small product catalog with LanceDB, filter it, search it by vector similarity and by keyword, modify it, and look back at earlier versions. - -## Install LanceDB - -This guide displays results as pandas DataFrames, so install pandas as well: - -```bash ---8<-- "snippets/quickstart/lancedb-install.sh" -``` - -## Connect to a Database - -This guide uses LanceDB in embedded mode, where a database is a directory of Lance tables, on local disk or in object storage such as `s3://`. Connect to a local path; LanceDB creates it if it doesn't exist: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:connect" -``` - -The same client connects to a distributed LanceDB deployment when you pass a `db://` URI instead of a path. - -## Create a Table - -Create a table from a list of dictionaries. LanceDB infers the schema and stores the `vector` column as a fixed-size vector: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:create" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.create.txt" -``` - -LanceDB also accepts pandas DataFrames, Polars DataFrames, and Arrow tables as input. - -## Filter Rows - -Use SQL expressions to filter rows and choose columns: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:filter" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.filter.txt" -``` - -## Search by Vector Similarity - -Pass a query vector to `search` to find the nearest rows. The `_distance` column holds each row's distance from the query: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:vector-search" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.vector-search.txt" -``` - -You can combine vector search with a filter: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:filtered-vector-search" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.filtered-vector-search.txt" -``` - -On a small table like this one, LanceDB compares the query against every vector. For larger tables, create a [vector index](https://docs.lancedb.com/indexing) to keep searches fast. - -## Search by Keyword - -Create a full-text search index on a text column, then search it with keywords. The `_score` column holds each row's relevance score: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:fts" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.fts.txt" -``` - -## Add, Update, and Delete Rows - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:modify" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.modify.txt" -``` - -## Time Travel - -Every change creates a new version of the table: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:version" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.version.txt" -``` - -Earlier versions stay readable. Check out version 1 to see the table as it was when you created it: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:checkout" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.checkout.txt" -``` - -Return to the latest version to continue making changes: - -```python ---8<-- "snippets/quickstart/lancedb_quickstart.py:checkout-latest" -``` - -```output ---8<-- "snippets/quickstart/outputs/lancedb_quickstart.py.checkout-latest.txt" -``` - -## Next Steps - -- Read the [LanceDB documentation](https://docs.lancedb.com/) for embeddings, hybrid search, reranking, distributed deployments, and the TypeScript and Rust SDKs. -- Explore the native format SDKs for [Python (PyLance)](pylance.md), [Rust](rust.md), and [Java](java.md). -- See the [Spark quickstart](spark.md) to process Lance tables at scale with SQL. diff --git a/docs/src/quickstart/pylance.md b/docs/src/quickstart/pylance.md index 56b8c1dc6d8..65e9e047df9 100644 --- a/docs/src/quickstart/pylance.md +++ b/docs/src/quickstart/pylance.md @@ -5,7 +5,7 @@ description: Get started with PyLance, the Python SDK for the Lance format - cre # Getting Started with PyLance -PyLance is the Python SDK for the Lance format itself. It gives you direct, low-level control over Lance datasets: their files, versions, and indices. If you are building an application or a data pipeline, start with [LanceDB](lancedb.md) or [Apache Spark](spark.md) instead. Both build on the same format and handle more of the work for you. +PyLance is the Python SDK for the Lance format itself. It gives you direct, low-level control over Lance datasets: their files, versions, and indices. If you are building an application or a data pipeline, start with [LanceDB](index.md) or [Apache Spark](spark.md) instead. Both build on the same format and handle more of the work for you. This quickstart guide will walk you through the core features of Lance including creating datasets, versioning, and vector search. diff --git a/docs/src/quickstart/rust.md b/docs/src/quickstart/rust.md index 4a4d42ccc83..5bce9d9a85e 100644 --- a/docs/src/quickstart/rust.md +++ b/docs/src/quickstart/rust.md @@ -5,7 +5,7 @@ description: Get started with the Lance Rust SDK — write Arrow batches, filter # Getting Started with Rust -The `lance` crate is the native Rust core and SDK for the Lance format. Use it when building Rust services, storage engines, or data pipelines that read and write Lance datasets directly with Apache Arrow. If you want a higher-level retrieval database API in Rust, see [LanceDB](lancedb.md) instead. +The `lance` crate is the native Rust SDK for the Lance format itself, giving direct, low-level control over Lance datasets, files, and indices. If you are building an application or analytics pipeline, start with an engine such as [LanceDB](index.md) (which also provides a Rust client) or [Apache Spark](spark.md) instead. ## Add Dependencies diff --git a/docs/src/quickstart/spark.md b/docs/src/quickstart/spark.md index cabe0293c61..3b04bf7d65e 100644 --- a/docs/src/quickstart/spark.md +++ b/docs/src/quickstart/spark.md @@ -162,5 +162,5 @@ Spark wrote an ordinary Lance table, so other engines can use it directly. For e ## Next Steps - Read the [Spark connector documentation](../integrations/spark/index.md) for the full set of supported DDL, DML, and query operations. -- See the [LanceDB quickstart](lancedb.md) to build search into an application. +- See the [LanceDB quickstart](index.md) to build search into an application. - See the [PyLance quickstart](pylance.md) to work directly with the Lance format. diff --git a/docs/src/sdk_docs.md b/docs/src/sdk_docs.md index 8b42a66e86a..446c74c5e49 100644 --- a/docs/src/sdk_docs.md +++ b/docs/src/sdk_docs.md @@ -7,3 +7,5 @@ all classes, functions, and methods available in each language. - [Python](https://lance-format.github.io/lance-python-doc) - [Rust](https://docs.rs/lance/latest/lance) - [Java](https://www.javadoc.io/doc/org.lance/lance-core/latest/index.html) + +For runnable introductory examples, see the [PyLance](quickstart/pylance.md), [Rust](quickstart/rust.md), and [Java](quickstart/java.md) SDK quickstarts. diff --git a/docs/tests/test_theme.py b/docs/tests/test_theme.py index 25316bc8196..6ff326bfb55 100644 --- a/docs/tests/test_theme.py +++ b/docs/tests/test_theme.py @@ -19,11 +19,12 @@ def setUpClass(cls): root = Path(tmp) for name in ( "index.md", + "qs/index.md", + "qs/spark.md", + "qs/pylance.md", + "qs/rust.md", "flat_guide/read.md", "flat_guide/search.md", - "guide/read_and_write.md", - "guide/spark/index.md", - "guide/spark/ddl/create.md", "spark/index.md", "spark/ddl/create.md", "trino/ddl/create.md", @@ -36,21 +37,17 @@ def setUpClass(cls): config = load_config( config_file=StringIO(""" nav: + - Getting Started: + - LanceDB: qs/index.md + - Apache Spark: qs/spark.md + - PyLance: + - Getting Started: qs/pylance.md + - Rust: qs/rust.md - User Guide: - Core Operations: - Read & Write: flat_guide/read.md - Indexing & Search: - Overview: flat_guide/search.md - - Track Guide: - - PyLance: - - Core Operations: - - Read and Write: guide/read_and_write.md - - LanceDB: https://docs.lancedb.com/ - - Apache Spark: - - Welcome: guide/spark/index.md - - Operations: - - DDL: - - CREATE NAMESPACE: guide/spark/ddl/create.md - Integrations: - External Docs: https://docs.lancedb.com/ - Overview: index.md @@ -75,9 +72,9 @@ def setUpClass(cls): cls.sidebars = {} cls.topnavs = {} for name in ( + "qs/index.html", + "qs/pylance/index.html", "flat_guide/read/index.html", - "guide/read_and_write/index.html", - "guide/spark/ddl/create/index.html", "spark/ddl/create/index.html", "standalone/index.html", ): @@ -138,16 +135,15 @@ def test_top_level_page(self): sidebar.find("./div/a").attrib, {"class": "active", "href": "./"} ) tabs = self.topnavs["standalone/index.html"].findall("a") - self.assertEqual(tabs[0].text, "User Guide") - self.assertEqual(tabs[0].get("href"), "../flat_guide/read/") - self.assertEqual(tabs[1].text, "Track Guide") - self.assertEqual(tabs[1].get("href"), "../guide/read_and_write/") + self.assertEqual(tabs[0].text, "Getting Started") + self.assertEqual(tabs[0].get("href"), "../qs/") + self.assertEqual(tabs[1].text, "User Guide") + self.assertEqual(tabs[1].get("href"), "../flat_guide/read/") self.assertEqual(tabs[2].text, "Integrations") self.assertEqual(tabs[2].get("href"), "..") - def test_two_level_guide_renders_flat_groups_without_switcher(self): + def test_two_level_guide_renders_flat_groups(self): sidebar = self.sidebars["flat_guide/read/index.html"] - self.assertIsNone(sidebar.find(".//div[@class='ld-sidenav__tracks']")) groups = sidebar.findall("./div[@class='ld-sidenav__group']") self.assertEqual( [group.find("div").text for group in groups], @@ -156,31 +152,21 @@ def test_two_level_guide_renders_flat_groups_without_switcher(self): self.assertEqual(groups[0].find("a").attrib.get("class"), "active") self.assertEqual(groups[0].find("a").text, "Read & Write") - def test_track_switcher_prioritizes_pylance_and_lancedb_and_scopes_tree(self): - pylance_sidebar = self.sidebars["guide/read_and_write/index.html"] - tracks = pylance_sidebar.findall("./div/div[@class='ld-sidenav__tracks']/a") + def test_mixed_section_keeps_engines_before_sdks_in_single_group(self): + qs_sidebar = self.sidebars["qs/pylance/index.html"] + groups = qs_sidebar.findall("./div[@class='ld-sidenav__group']") + self.assertEqual(len(groups), 1) + group = groups[0] + self.assertEqual(group.find("div").text, "Getting Started") self.assertEqual( - [a.find("span").text for a in tracks], - ["PyLance", "LanceDB", "Apache Spark"], + [child.tag for child in group], ["div", "a", "a", "details", "a"] ) - self.assertIn("active", tracks[0].attrib.get("class", "")) - self.assertEqual(tracks[1].attrib.get("href"), "https://docs.lancedb.com/") - groups = pylance_sidebar.findall("./div[@class='ld-sidenav__group']") - self.assertEqual(len(groups), 1) - self.assertEqual(groups[0].find("div").text, "Core Operations") - self.assertEqual(groups[0].find("a").text, "Read and Write") - - spark_sidebar = self.sidebars["guide/spark/ddl/create/index.html"] - spark_tracks = spark_sidebar.findall("./div/div[@class='ld-sidenav__tracks']/a") - self.assertIn("active", spark_tracks[2].attrib.get("class", "")) - spark_groups = spark_sidebar.findall("./div[@class='ld-sidenav__group']") - self.assertEqual(len(spark_groups), 2) - self.assertIsNone(spark_groups[0].find("./div[@class='ld-sidenav__label']")) - self.assertEqual(spark_groups[0].find("a").text, "Welcome") - self.assertEqual(spark_groups[1].find("div").text, "Operations") - ddl = spark_groups[1].find("./details[@class='ld-sidenav__sub']") - self.assertEqual(ddl.attrib.get("open"), "open") - self.assertEqual(ddl.find("./summary/span").text, "DDL") + links = group.findall("a") + self.assertEqual([a.text for a in links], ["LanceDB", "Apache Spark", "Rust"]) + pylance = group.find("./details[@class='ld-sidenav__sub']") + self.assertEqual(pylance.attrib.get("open"), "open") + self.assertEqual(pylance.find("./summary/span").text, "PyLance") + self.assertEqual(pylance.find("./div/a").attrib.get("class"), "active") def test_alternate_style_language_tabs_render_radio_inputs_and_labels(self): with TemporaryDirectory() as tmp: @@ -222,33 +208,3 @@ def test_alternate_style_language_tabs_render_radio_inputs_and_labels(self): self.assertIn(">Python", html) self.assertIn(">Rust", html) self.assertIn(">Java", html) - - def test_homepage_renders_three_layer_stack_links(self): - with TemporaryDirectory() as tmp: - root = Path(tmp) - home = root / "src" / "index.md" - home.parent.mkdir(parents=True, exist_ok=True) - home.write_text("---\ntemplate: home.html\n---\n", encoding="utf-8") - config = load_config( - config_file=StringIO("nav:\n - Home: index.md\n"), - site_name="Home test", - docs_dir=str(root / "src"), - site_dir=str(root / "site"), - theme={"name": None, "custom_dir": str(DOCS_DIR / "theme")}, - plugins=[], - ) - build(config) - html = (root / "site" / "index.html").read_text(encoding="utf-8") - self.assertIn("01 · Database", html) - self.assertIn("02 · Language SDKs", html) - self.assertIn("03 · Query & Compute Engines", html) - for expected_href in ( - 'href="quickstart/lancedb/"', - 'href="quickstart/pylance/"', - 'href="quickstart/rust/"', - 'href="quickstart/java/"', - 'href="integrations/spark/"', - 'href="format/"', - 'href="format/namespace/"', - ): - self.assertIn(expected_href, html) diff --git a/docs/theme/assets/site.css b/docs/theme/assets/site.css index 4364a828e8c..bb25eda15c3 100644 --- a/docs/theme/assets/site.css +++ b/docs/theme/assets/site.css @@ -262,92 +262,6 @@ .ld-feature__img { border: 1px solid var(--line-1); padding: 16px; background: #ffffff; } .ld-feature__img img { display: block; width: 100%; height: auto; } -/* homepage three-layer ecosystem stack */ -.ld-stack { - border: 1px solid var(--line-1); - border-radius: var(--radius-box); - background: var(--surface-card); - overflow: hidden; - min-width: 0; -} -.ld-stack__layer { - padding: 14px 18px 16px; -} -.ld-stack__layer + .ld-stack__layer { - border-top: 1px solid var(--line-2); -} -.ld-stack__meta { - font-family: var(--font-mono); - font-size: 11px; - font-weight: 600; - letter-spacing: var(--tracking-caps); - text-transform: uppercase; - color: var(--text-muted); - margin-bottom: 10px; -} -.ld-stack__chips { - display: flex; - flex-wrap: wrap; - gap: 8px; -} -.ld-stack__chip { - display: inline-flex; - align-items: center; - font-family: var(--font-mono); - font-size: 12px; - font-weight: 500; - color: var(--fg-1); - background: var(--surface-overlay); - border: 1px solid var(--line-2); - border-radius: var(--radius-chip); - padding: 5px 10px; - text-decoration: none; - transition: border-color var(--dur-fast) var(--ease-out), color var(--dur-fast) var(--ease-out); -} -.ld-stack__chip:hover { - border-color: var(--beam-400); - color: var(--beam-600); -} -.ld-stack__chip--primary { - border-color: var(--beam-400); - background: var(--beam-dim); - color: var(--beam-600); -} -.ld-stack__chip--primary:hover { - background: var(--beam-400); - color: #ffffff; -} -.ld-stack__base { - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-wrap: wrap; - padding: 11px 18px; - border-top: 1px solid var(--line-1); - background: var(--surface-overlay); - font-family: var(--font-mono); - font-size: 11px; - letter-spacing: 0.06em; - text-transform: uppercase; - color: var(--text-muted); -} -.ld-stack__base-links { - display: inline-flex; - align-items: center; - gap: 8px; - text-transform: none; - letter-spacing: 0.02em; - font-size: 12px; -} -.ld-stack__base-links a { - color: var(--beam-600); - text-decoration: none; -} -.ld-stack__base-links a:hover { - text-decoration: underline; -} - /* homepage code windows (always ink-dark, like docs code blocks) */ .ld-feature__code { border: 1px solid var(--line-2); background: var(--ink-900); min-width: 0; border-radius: var(--radius-box); overflow: hidden; } .ld-win { display: flex; gap: 6px; padding: 12px 16px 0; } @@ -422,37 +336,6 @@ color: var(--text-muted); margin: 22px 0 8px; } -.ld-sidenav__group--tracks { margin-bottom: 18px; } -.ld-sidenav__tracks { - display: flex; - flex-direction: column; - border: 1px solid var(--line-2); - background: var(--surface-overlay); -} -.ld-sidenav a.ld-sidenav__track { - display: flex; - align-items: center; - justify-content: space-between; - padding: 7px 12px; - font-size: 13px; - font-weight: 500; - color: var(--text-secondary); - border-left: 2px solid transparent; - border-bottom: 1px solid var(--line-2); - transition: background var(--dur-fast) var(--ease-out), color var(--dur-fast) var(--ease-out); -} -.ld-sidenav a.ld-sidenav__track:last-child { border-bottom: 0; } -.ld-sidenav a.ld-sidenav__track:hover { - color: var(--fg-1); - background: var(--surface-card); -} -.ld-sidenav a.ld-sidenav__track.active { - color: var(--beam-600); - background: var(--surface-card); - border-left: 2px solid var(--beam-400); - padding-left: 12px; - font-weight: 600; -} .ld-sidenav a { display: block; font-size: 13.5px; diff --git a/docs/theme/home.html b/docs/theme/home.html index 8aca375336b..e647c262cef 100644 --- a/docs/theme/home.html +++ b/docs/theme/home.html @@ -134,47 +134,13 @@

Data evolution > schema evolution

05
-

One format across a three-layer stack

-

Every layer reads and writes the same Lance tables on object storage without copying data — start with LanceDB as your AI database, embed the Python, Rust, or Java SDKs directly, or connect SQL and ML compute engines across your lakehouse.

- View all integrations → +

Rich ecosystem integrations

+

Works with Pandas, Polars, Ray, and PyTorch for processing and ML. Connects to Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink for SQL analytics and distributed processing.

+ View integrations → +
+
+ Lance ecosystem integrations
-
diff --git a/docs/theme/main.html b/docs/theme/main.html index 96556e23470..54b29d0a883 100644 --- a/docs/theme/main.html +++ b/docs/theme/main.html @@ -1,12 +1,5 @@ {% extends "base.html" %} -{#- Resolve the first concrete page inside a nested navigation section. -#} -{%- macro track_first_url(item) -%} - {%- set found = namespace(matched=false, url=item.url) -%} - {{- find_first_page(item, found) -}} - {{- found.url -}} -{%- endmacro -%} - {#- Render leaf links and collapsible nested subsections inside a group. -#} {%- macro sidenav_item(item) -%} {%- if item.is_section %} @@ -40,22 +33,34 @@ {%- endmacro -%} {%- macro render_section_groups(children, root_label=none) -%} - {%- set direct = namespace(items=[], used_root=false) -%} + {%- set shape = namespace(seen_section=false, leaf_after_section=false) -%} {%- for item in children %} {%- if item.is_section %} - {%- if direct.items %} - {{ sidenav_group(direct.items, root_label if not direct.used_root else none) }} - {%- set direct.items = [] -%} + {%- set shape.seen_section = true -%} + {%- elif shape.seen_section %} + {%- set shape.leaf_after_section = true -%} + {%- endif %} + {%- endfor %} + {%- if shape.leaf_after_section %} + {{ sidenav_group(children, root_label) }} + {%- else %} + {%- set direct = namespace(items=[], used_root=false) -%} + {%- for item in children %} + {%- if item.is_section %} + {%- if direct.items %} + {{ sidenav_group(direct.items, root_label if not direct.used_root else none) }} + {%- set direct.items = [] -%} + {%- set direct.used_root = true -%} + {%- endif %} + {{ sidenav_group(item.children, item.title) }} {%- set direct.used_root = true -%} + {%- else %} + {%- set direct.items = direct.items + [item] -%} {%- endif %} - {{ sidenav_group(item.children, item.title) }} - {%- set direct.used_root = true -%} - {%- else %} - {%- set direct.items = direct.items + [item] -%} + {%- endfor %} + {%- if direct.items %} + {{ sidenav_group(direct.items, root_label if not direct.used_root else none) }} {%- endif %} - {%- endfor %} - {%- if direct.items %} - {{ sidenav_group(direct.items, root_label if not direct.used_root else none) }} {%- endif %} {%- endmacro -%} @@ -70,50 +75,7 @@