Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions java/src/main/java/org/lance/index/OptimizeOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,11 @@ public Optional<List<String>> getIndexNames() {
return indexNames;
}

/** Whether to retrain the index instead of performing an incremental merge. */
/**
* Whether to retrain the index instead of performing an incremental merge.
*
* <p>Only vector indices are retrained; scalar and full-text search indices are not.
*/
public boolean isRetrain() {
return retrain;
}
Expand Down Expand Up @@ -140,7 +144,8 @@ public Builder indexNames(List<String> indexNames) {
}

/**
* Whether to retrain the index.
* Whether to retrain the index. Only vector indices are retrained; see {@link
* OptimizeOptions#isRetrain()}.
*
* @param retrain if true, retrain instead of incremental merge
*/
Expand Down
28 changes: 23 additions & 5 deletions python/python/lance/_dataset/optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,16 @@ def inline_deletions(self) -> int:
"""
return self._dataset._ds.inline_deletions()

def optimize_indices(self, **kwargs) -> None:
def optimize_indices(
self,
*,
num_indices_to_merge: Optional[int] = None,
index_names: Optional[list[str]] = None,
retrain: Optional[bool] = None,
max_rows_per_task: Optional[int] = None,
max_rows_per_segment: Optional[int] = None,
max_rows_to_consolidate: Optional[int] = None,
) -> None:
"""Optimizes index performance.

As new data arrives it is not added to existing indexes automatically.
Expand All @@ -300,18 +309,20 @@ def optimize_indices(self, **kwargs) -> None:
num_indices_to_merge: optional, int, default None
The number of indices to merge.
If set to 0, new delta index will be created.
index_names: List[str], default None
index_names: optional, list[str], default None
The names of the indices to optimize.
If None, all indices will be optimized.
retrain: bool, default False
Whether to retrain the whole index.
retrain: optional, bool, default None
Whether to retrain the whole index. None is the same as False.
If true, the index will be retrained based on the current data,
`num_indices_to_merge` will be ignored,
and all indices will be merged into one.

This is useful when the data distribution has changed significantly,
and we want to retrain the index to improve the search quality.
This rebuilds the index from the source data and may be expensive.
Only vector indices are retrained; scalar and full-text search
indices are not.
max_rows_per_task: optional, int, default None
Upper bound on the rows one build task covers. Setting this or
`max_rows_per_segment` splits the build of every index whose
Expand Down Expand Up @@ -339,7 +350,14 @@ def optimize_indices(self, **kwargs) -> None:
belong to :py:meth:`lance.optimize.OptimizeIndices.commit`, which uses
the defaults when they are omitted.
"""
self._dataset._ds.optimize_indices(**kwargs)
self._dataset._ds.optimize_indices(
num_indices_to_merge=num_indices_to_merge,
index_names=index_names,
retrain=retrain,
max_rows_per_task=max_rows_per_task,
max_rows_per_segment=max_rows_per_segment,
max_rows_to_consolidate=max_rows_to_consolidate,
)

def enable_auto_cleanup(self, auto_cleanup_config: AutoCleanupConfig, **kwargs):
"""Enable autocleaning for an existing dataset.
Expand Down
27 changes: 27 additions & 0 deletions python/python/tests/test_optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -1535,6 +1535,33 @@ def test_min_bytes_per_file(tmp_path: Path, min_bytes):
assert metrics.fragments_removed == 2


def test_optimize_indices_rejects_unknown_options(tmp_path: Path):
"""An option name optimize_indices does not know must raise. Ignored, a
misspelled ``index_names`` runs the default optimize over every index."""
uri = str(tmp_path / "dataset")
dataset = lance.write_dataset(pa.table({"id": range(64)}), uri)
dataset.create_scalar_index("id", "BTREE", name="id_idx")
dataset = lance.write_dataset(pa.table({"id": range(64, 128)}), uri, mode="append")
version = dataset.version

with pytest.raises(TypeError, match="index_name"):
dataset.optimize.optimize_indices(index_name=["id_idx"])
with pytest.raises(ValueError, match="Invalid optimize_indices option: index_name"):
dataset._ds.optimize_indices(index_name=["id_idx"])
with pytest.raises(ValueError, match="Invalid optimize_indices option: index_name"):
OptimizeIndices.plan(dataset, {"index_name": ["id_idx"]})
with pytest.raises(ValueError, match="Invalid optimize_indices option: index_name"):
OptimizeIndices.commit(dataset, [], {"index_name": ["id_idx"]})
assert dataset.version == version
assert lance.dataset(uri).version == version

# A known option set to None is unset, in every entry point alike.
plan = OptimizeIndices.plan(dataset, {"index_names": None, "retrain": None})
assert [task.index_name for task in plan.tasks] == ["id_idx"]
dataset.optimize.optimize_indices(index_names=["id_idx"], retrain=None)
assert dataset.version == version + 1


def test_optimize_indices_distributed(tmp_path: Path):
"""The plan's tasks and merges travel to worker processes by pickle and
their results travel back, while the table keeps taking writes."""
Expand Down
55 changes: 32 additions & 23 deletions python/src/dataset/optimize_indices.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,34 +15,43 @@ use serde::{Serialize, de::DeserializeOwned};
use super::optimize::unwrap_dataset;
use super::*;

/// The `OptimizeOptions` of the keyword options of `optimize_indices`, read
/// as it always has: a key it does not know is ignored.
/// The `OptimizeOptions` of the keyword options of `optimize_indices`, a
/// `None` value leaving the option unset. A key it does not know is rejected:
/// ignoring it would run the default optimize over every index instead.
pub(crate) fn optimize_options(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<OptimizeOptions> {
let mut options = OptimizeOptions::default();
let Some(kwargs) = kwargs else {
return Ok(options);
};
if let Some(num_indices_to_merge) = kwargs.get_item("num_indices_to_merge")? {
options.num_indices_to_merge = num_indices_to_merge.extract()?;
}
if let Some(retrain) = kwargs.get_item("retrain")? {
options.retrain = retrain.extract()?;
}
if let Some(index_names) = kwargs.get_item("index_names")? {
options.index_names = Some(
index_names
.extract::<Vec<String>>()
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
}
if let Some(max_rows_per_task) = kwargs.get_item("max_rows_per_task")? {
options.max_rows_per_task = max_rows_per_task.extract()?;
}
if let Some(max_rows_per_segment) = kwargs.get_item("max_rows_per_segment")? {
options.max_rows_per_segment = max_rows_per_segment.extract()?;
}
if let Some(max_rows_to_consolidate) = kwargs.get_item("max_rows_to_consolidate")? {
options.max_rows_to_consolidate = max_rows_to_consolidate.extract()?;
for (key, value) in kwargs.iter() {
let key: String = key.extract()?;
match key.as_str() {
"num_indices_to_merge" => {
options.num_indices_to_merge = value.extract()?;
}
"retrain" => {
options.retrain = value.extract::<Option<bool>>()?.unwrap_or_default();
}
"index_names" => {
options.index_names = value
.extract::<Option<Vec<String>>>()
.map_err(|err| PyValueError::new_err(err.to_string()))?;
}
"max_rows_per_task" => {
options.max_rows_per_task = value.extract()?;
}
"max_rows_per_segment" => {
options.max_rows_per_segment = value.extract()?;
}
"max_rows_to_consolidate" => {
options.max_rows_to_consolidate = value.extract()?;
}
_ => {
return Err(PyValueError::new_err(format!(
"Invalid optimize_indices option: {key}"
)));
}
}
}
Ok(options)
}
Expand Down
5 changes: 3 additions & 2 deletions rust/lance-index/src/optimize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ pub struct OptimizeOptions {
///
/// This is useful when the data distribution has changed significantly,
/// and we want to retrain the index to improve the search quality.
/// This would be faster than re-create the index from scratch.
/// The index is rebuilt from the source data.
///
/// NOTE: this option is only supported for v3 vector indices.
/// NOTE: only vector indices are retrained; scalar and full-text search
/// indices are not.
pub retrain: bool,

/// Transaction properties to store with this commit.
Expand Down
Loading