Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 66 additions & 3 deletions rust/lance-io/src/object_store.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

Expand Down Expand Up @@ -1382,6 +1382,69 @@
}
}

/// Read an object directly, preserving request and body errors separately.
pub async fn read_object(&self, path: &Path) -> object_store::Result<object_store::GetResult> {
self.inner.get(path).await
}

/// Publish a payload without buffering it through a Lance writer.
pub async fn put_payload(
&self,
path: &Path,
payload: PutPayload,
) -> object_store::Result<WriteResult> {
self.put_payload_opts(path, payload, Default::default())
.await
}

/// Publish a payload with the caller's preconditions.
pub async fn put_payload_opts(
&self,
path: &Path,
payload: PutPayload,
options: object_store::PutOptions,
) -> object_store::Result<WriteResult> {
let size = payload.content_length();
let result = self.inner.put_opts(path, payload, options).await?;
Ok(WriteResult {
size,
e_tag: result.e_tag,
})
}

/// List objects with a single backend stream.
pub fn list_objects(
&self,
prefix: Option<&Path>,
) -> BoxStream<'static, object_store::Result<ObjectMeta>> {
self.inner.list(prefix)
}

/// Read one byte range in a single backend request.
pub async fn read_range(&self, path: &Path, range: Range<u64>) -> object_store::Result<Bytes> {
self.inner.get_range(path, range).await
}

/// Read metadata without fetching the object body.
pub async fn stat(&self, path: &Path) -> object_store::Result<ObjectMeta> {
self.inner.head(path).await
}

/// Rename an object using the provider's replacement semantics.
pub async fn rename(&self, from: &Path, to: &Path) -> object_store::Result<()> {
self.inner.rename(from, to).await
}

/// Publish by rename only if the destination is absent.
pub async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> object_store::Result<()> {
self.inner.rename_if_not_exists(from, to).await
}

/// Copy only if the destination is absent.
pub async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> object_store::Result<()> {
self.inner.copy_if_not_exists(from, to).await
}

/// A helper function to create a file and write content to it.
pub async fn put(&self, path: &Path, content: &[u8]) -> Result<WriteResult> {
let mut writer = self.create(path).await?;
Expand Down Expand Up @@ -1665,7 +1728,7 @@
///
/// This uses the provider's native conditional copy. An existing destination
/// with the same size is treated as a successful retry.
pub async fn copy_if_not_exists(
pub async fn copy_if_not_exists_or_same_size(
&self,
source_path: &Path,
destination_path: &Path,
Expand Down Expand Up @@ -4315,7 +4378,7 @@
store.put(&destination, b"old").await.unwrap();

let result = store
.copy_if_not_exists(&source, &destination)
.copy_if_not_exists_or_same_size(&source, &destination)
.await
.unwrap();

Expand Down Expand Up @@ -4344,7 +4407,7 @@
observations.part_count.store(0, Ordering::SeqCst);

let result = store
.copy_if_not_exists(&source, &destination)
.copy_if_not_exists_or_same_size(&source, &destination)
.await
.unwrap();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ mod tests {
.await;
let store = provider_store(&server, mode).await;
let result = store
.copy_if_not_exists(&Path::from("source"), &Path::from("destination"))
.copy_if_not_exists_or_same_size(&Path::from("source"), &Path::from("destination"))
.await
.unwrap();
assert_eq!(result.size, 3);
Expand All @@ -289,7 +289,7 @@ mod tests {
.await;
provider_store(&server, Some("header:cf-copy-destination-if-none-match:*"))
.await
.copy_if_not_exists(&Path::from("source"), &Path::from("destination"))
.copy_if_not_exists_or_same_size(&Path::from("source"), &Path::from("destination"))
.await
.unwrap();
}
Expand All @@ -307,7 +307,7 @@ mod tests {
.await;
let result = provider_store(&server, None)
.await
.copy_if_not_exists(&Path::from("source"), &Path::from("destination"))
.copy_if_not_exists_or_same_size(&Path::from("source"), &Path::from("destination"))
.await
.unwrap();
assert_eq!(result.size, 0);
Expand Down Expand Up @@ -338,7 +338,7 @@ mod tests {
.await;
let error = provider_store(&server, mode)
.await
.copy_if_not_exists(&Path::from("source"), &Path::from("destination"))
.copy_if_not_exists_or_same_size(&Path::from("source"), &Path::from("destination"))
.await
.unwrap_err();
assert!(matches!(error, lance_core::Error::IO { .. }));
Expand Down
26 changes: 7 additions & 19 deletions rust/lance-namespace-impls/src/dir/marker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@

use lance_io::object_store::ObjectStore;
use object_store::path::Path;
use object_store::{
Error as ObjectStoreError, ObjectStore as OSObjectStore, ObjectStoreExt, PutMode, PutOptions,
};
use object_store::{Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions};

/// Error from [`put_marker_file_atomic`].
#[derive(Debug)]
Expand Down Expand Up @@ -56,8 +54,7 @@ pub async fn put_marker_file_atomic(
) -> std::result::Result<(), MarkerFileError> {
let publish_result = if matches!(object_store.scheme(), "s3" | "s3+ddb") {
object_store
.inner
.put_opts(
.put_payload_opts(
path,
bytes::Bytes::from_static(b"reserved").into(),
PutOptions {
Expand All @@ -77,8 +74,7 @@ pub async fn put_marker_file_atomic(
};

object_store
.inner
.put(&staging_path, bytes::Bytes::from_static(b"reserved").into())
.put_payload(&staging_path, bytes::Bytes::from_static(b"reserved").into())
.await
.map_err(|e| MarkerFileError::Other {
message: format!("Failed to stage {}: {:?}", file_description, e),
Expand All @@ -87,27 +83,18 @@ pub async fn put_marker_file_atomic(
// Successful rename consumes the staging object; every other path must
// delete it (best-effort) so conflict/fallback races do not accumulate.
let mut staging_consumed = false;
let publish_result = match object_store
.inner
.rename_if_not_exists(&staging_path, path)
.await
{
let publish_result = match object_store.rename_if_not_exists(&staging_path, path).await {
Ok(()) => {
staging_consumed = true;
Ok(())
}
Err(ObjectStoreError::NotImplemented { .. })
| Err(ObjectStoreError::NotSupported { .. }) => {
match object_store
.inner
.copy_if_not_exists(&staging_path, path)
.await
{
match object_store.copy_if_not_exists(&staging_path, path).await {
Ok(()) => Ok(()),
Err(ObjectStoreError::NotImplemented { .. })
| Err(ObjectStoreError::NotSupported { .. }) => object_store
.inner
.put_opts(
.put_payload_opts(
path,
bytes::Bytes::from_static(b"reserved").into(),
PutOptions {
Expand Down Expand Up @@ -174,6 +161,7 @@ async fn delete_staging_marker_best_effort(object_store: &ObjectStore, staging_p
#[cfg(test)]
mod tests {
use super::*;
use object_store::ObjectStore as OSObjectStore;

use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
Expand Down
25 changes: 10 additions & 15 deletions rust/lance-namespace-impls/src/dir/namespace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,7 @@ impl LanceNamespace for DirectoryNamespace {
})?;

// Idempotent retry: version path already published with the same content.
match self.object_store.inner.head(&final_path).await {
match self.object_store.stat(&final_path).await {
Ok(existing_meta) => {
return self
.resolve_existing_table_version(ExistingTableVersionResolve {
Expand Down Expand Up @@ -884,7 +884,7 @@ impl LanceNamespace for DirectoryNamespace {
Err(ObjectStoreError::AlreadyExists { .. })
| Err(ObjectStoreError::Precondition { .. }) => {
// Lost a Create race: succeed only if the winner published identical bytes.
let existing_meta = self.object_store.inner.head(&final_path).await.map_err(|e| {
let existing_meta = self.object_store.stat(&final_path).await.map_err(|e| {
lance_core::Error::from(NamespaceError::Internal {
message: format!(
"Version {} conflict for table at '{}' but failed to stat winner: {}",
Expand Down Expand Up @@ -921,19 +921,14 @@ impl LanceNamespace for DirectoryNamespace {
}
}

let final_meta = self
.object_store
.inner
.head(&final_path)
.await
.map_err(|e| {
lance_core::Error::from(NamespaceError::Internal {
message: format!(
"Failed to stat created version {} for table at '{}': {}",
version, table_uri, e
),
})
})?;
let final_meta = self.object_store.stat(&final_path).await.map_err(|e| {
lance_core::Error::from(NamespaceError::Internal {
message: format!(
"Failed to stat created version {} for table at '{}': {}",
version, table_uri, e
),
})
})?;

// Delete the staging manifest after successful copy
if let Err(e) = self.object_store.inner.delete(&staging_path).await {
Expand Down
19 changes: 7 additions & 12 deletions rust/lance-namespace-impls/src/dir/tests/branches.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,7 @@ async fn test_create_table_version_on_branch() {
let versions_dir = branch_ds.versions_dir();
let store = branch_ds.object_store(None).await.unwrap();
let existing = store
.inner
.list(Some(&versions_dir))
.list_objects(Some(&versions_dir))
.try_collect::<Vec<_>>()
.await
.unwrap()
Expand All @@ -207,15 +206,14 @@ async fn test_create_table_version_on_branch() {
})
.expect("a branch manifest");
let bytes = store
.inner
.get(&existing.location)
.read_object(&existing.location)
.await
.unwrap()
.bytes()
.await
.unwrap();
let staging = versions_dir.join("staging_manifest");
store.inner.put(&staging, bytes.into()).await.unwrap();
store.put_payload(&staging, bytes.into()).await.unwrap();

let main_before = list_versions(&namespace, "users", None)
.await
Expand Down Expand Up @@ -569,8 +567,7 @@ async fn test_branch_ops_reject_zombie_branch() {
let dataset = open_dataset(&namespace, "users").await;
let store = dataset.object_store(None).await.unwrap();
let manifest = store
.inner
.list(Some(&dataset.versions_dir()))
.list_objects(Some(&dataset.versions_dir()))
.try_collect::<Vec<_>>()
.await
.unwrap()
Expand All @@ -583,8 +580,7 @@ async fn test_branch_ops_reject_zombie_branch() {
})
.expect("a manifest");
let bytes = store
.inner
.get(&manifest.location)
.read_object(&manifest.location)
.await
.unwrap()
.bytes()
Expand All @@ -597,7 +593,7 @@ async fn test_branch_ops_reject_zombie_branch() {
.path
.join(VERSIONS_DIR)
.join(manifest.location.filename().unwrap());
store.inner.put(&zombie, bytes.into()).await.unwrap();
store.put_payload(&zombie, bytes.into()).await.unwrap();

assert!(dataset.branches().get("ghost").await.is_err());

Expand Down Expand Up @@ -1216,8 +1212,7 @@ async fn test_managed_branch_open_and_commit() {
let store = branch_ds.object_store(None).await.unwrap();
let branch_data = branch_base.clone().join("data");
let branch_files = store
.inner
.list(Some(&branch_data))
.list_objects(Some(&branch_data))
.try_collect::<Vec<_>>()
.await
.unwrap();
Expand Down
12 changes: 4 additions & 8 deletions rust/lance-namespace-impls/src/dir/tests/table_versions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -897,8 +897,7 @@ async fn test_create_table_version_branch_cas_requires_parent_version() {
let versions_dir = branch_ds.versions_dir();
let store = branch_ds.object_store(None).await.unwrap();
let manifests: Vec<_> = store
.inner
.list(Some(&versions_dir))
.list_objects(Some(&versions_dir))
.try_collect()
.await
.unwrap();
Expand All @@ -914,8 +913,7 @@ async fn test_create_table_version_branch_cas_requires_parent_version() {
// Confirm the branch object-store chain is empty (do not open the dataset:
// with no manifests, Dataset::open would fail).
let remaining_manifests = store
.inner
.list(Some(&versions_dir))
.list_objects(Some(&versions_dir))
.try_collect::<Vec<_>>()
.await
.unwrap()
Expand Down Expand Up @@ -959,8 +957,7 @@ async fn test_create_table_version_branch_cas_requires_parent_version() {

let staging_wrong = versions_dir.clone().join("staging_wrong");
store
.inner
.put(&staging_wrong, source_bytes.clone().into())
.put_payload(&staging_wrong, source_bytes.clone().into())
.await
.unwrap();
let err = namespace
Expand All @@ -982,8 +979,7 @@ async fn test_create_table_version_branch_cas_requires_parent_version() {

let staging_ok = versions_dir.join("staging_ok");
store
.inner
.put(&staging_ok, source_bytes.into())
.put_payload(&staging_ok, source_bytes.into())
.await
.unwrap();
let resp = namespace
Expand Down
7 changes: 3 additions & 4 deletions rust/lance-namespace-impls/src/dir/transactions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use lance::dataset::transaction::{Operation, Transaction};
use lance_core::Result;
use lance_namespace::error::NamespaceError;
use lance_namespace::models::DescribeTransactionResponse;
use object_store::Error as ObjectStoreError;
use object_store::path::Path;
use object_store::{Error as ObjectStoreError, ObjectStoreExt};

use super::DirectoryNamespace;

Expand Down Expand Up @@ -220,7 +220,7 @@ impl DirectoryNamespace {
txn_uuid: &str,
) -> Result<Option<TransactionAlteration>> {
let path = self.transaction_alteration_path(table_uri, txn_uuid)?;
match self.object_store.inner.get(&path).await {
match self.object_store.read_object(&path).await {
Ok(get_result) => {
let bytes = get_result.bytes().await.map_err(|e| {
lance_core::Error::from(NamespaceError::Internal {
Expand Down Expand Up @@ -266,8 +266,7 @@ impl DirectoryNamespace {
})
})?;
self.object_store
.inner
.put(&path, bytes.into())
.put_payload(&path, bytes.into())
.await
.map_err(|e| {
lance_core::Error::from(NamespaceError::Internal {
Expand Down
Loading
Loading