quintodrome/plugins/pdk/rust/nd-pdk-host/src/nd_host_kvstore.rs
Deluan Quintão 2471bb9cf6
feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)
* feat(plugins): add expires_at column to kvstore schema

* feat(plugins): filter expired keys in kvstore Get, Has, List

* feat(plugins): add periodic cleanup of expired kvstore keys

* feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore

Add three new methods to the KVStore host service:

- SetWithTTL: store key-value pairs with automatic expiration
- DeleteByPrefix: remove all keys matching a prefix in one operation
- GetMany: retrieve multiple values in a single call

All methods include comprehensive unit tests covering edge cases,
expiration behavior, size tracking, and LIKE-special characters.

* feat(plugins): regenerate code and update test plugin for new kvstore methods

Regenerate host function wrappers and PDK bindings for Go, Python,
and Rust. Update the test-kvstore plugin to exercise SetWithTTL,
DeleteByPrefix, and GetMany.

* feat(plugins): add integration tests for new kvstore methods

Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany
operations through the plugin boundary, verifying end-to-end behavior
including TTL expiration, prefix deletion, and batch retrieval.

* fix(plugins): address lint issues in kvstore implementation

Handle tx.Rollback error return and suppress gosec false positive
for parameterized SQL query construction in GetMany.

* fix(plugins): Set clears expires_at when overwriting a TTL'd key

Previously, calling Set() on a key that was stored with SetWithTTL()
would leave the expires_at value intact, causing the key to silently
expire even though Set implies permanent storage.

Also excludes expired keys from currentSize calculation at startup.

* refactor(plugins): simplify kvstore by removing in-memory size cache

Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup
timer, and mutex with direct database queries for storage accounting.
This eliminates race conditions and cache drift issues at negligible
performance cost for plugin-sized datasets. Also unified Set and
SetWithTTL into a shared setValue method, simplified DeleteByPrefix to
use RowsAffected instead of a transaction, and added an index on
expires_at for efficient expiration filtering.

* feat(plugins): add generic SQLite migration helper and refactor kvstore schema

Add a reusable migrateDB helper that tracks schema versions via SQLite's
PRAGMA user_version and applies pending migrations transactionally. Replace
the ad-hoc createKVStoreSchema function in kvstore with a declarative
migrations slice, making it easy to add future schema changes. Remove the
now-redundant schema migration test since migrateDB has its own test suite
and every kvstore test exercises the migrations implicitly.

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout

- Use sql.NullString for expires_at to explicitly send NULL instead of
  relying on datetime('now', '') returning NULL by accident
- Reject empty prefix in DeleteByPrefix to prevent accidental data wipe
- Add 5s timeout context to cleanupExpired on Close
- Replace time.Sleep in unit tests with pre-expired timestamps

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(plugins): use batch processing in GetMany

Process keys in chunks of 200 using slice.CollectChunks to avoid
hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets.

* feat(plugins): add periodic cleanup goroutine for expired kvstore keys

Use the manager's context to control a background goroutine that purges
expired keys every hour, stopping naturally on shutdown when the context
is cancelled.

---------

Signed-off-by: Deluan <deluan@navidrome.org>
2026-02-28 23:12:17 -05:00

433 lines
11 KiB
Rust

// Code generated by ndpgen. DO NOT EDIT.
//
// This file contains client wrappers for the KVStore host service.
// It is intended for use in Navidrome plugins built with extism-pdk.
use extism_pdk::*;
use serde::{Deserialize, Serialize};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
mod base64_bytes {
use serde::{self, Deserialize, Deserializer, Serializer};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&BASE64.encode(bytes))
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
BASE64.decode(&s).map_err(serde::de::Error::custom)
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreSetRequest {
key: String,
#[serde(with = "base64_bytes")]
value: Vec<u8>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreSetResponse {
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreSetWithTTLRequest {
key: String,
#[serde(with = "base64_bytes")]
value: Vec<u8>,
ttl_seconds: i64,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreSetWithTTLResponse {
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreGetRequest {
key: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreGetResponse {
#[serde(default)]
#[serde(with = "base64_bytes")]
value: Vec<u8>,
#[serde(default)]
exists: bool,
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreGetManyRequest {
keys: Vec<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreGetManyResponse {
#[serde(default)]
values: std::collections::HashMap<String, Vec<u8>>,
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreHasRequest {
key: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreHasResponse {
#[serde(default)]
exists: bool,
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreListRequest {
prefix: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreListResponse {
#[serde(default)]
keys: Vec<String>,
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreDeleteRequest {
key: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreDeleteResponse {
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreDeleteByPrefixRequest {
prefix: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreDeleteByPrefixResponse {
#[serde(default)]
deleted_count: i64,
#[serde(default)]
error: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KVStoreGetStorageUsedResponse {
#[serde(default)]
bytes: i64,
#[serde(default)]
error: Option<String>,
}
#[host_fn]
extern "ExtismHost" {
fn kvstore_set(input: Json<KVStoreSetRequest>) -> Json<KVStoreSetResponse>;
fn kvstore_setwithttl(input: Json<KVStoreSetWithTTLRequest>) -> Json<KVStoreSetWithTTLResponse>;
fn kvstore_get(input: Json<KVStoreGetRequest>) -> Json<KVStoreGetResponse>;
fn kvstore_getmany(input: Json<KVStoreGetManyRequest>) -> Json<KVStoreGetManyResponse>;
fn kvstore_has(input: Json<KVStoreHasRequest>) -> Json<KVStoreHasResponse>;
fn kvstore_list(input: Json<KVStoreListRequest>) -> Json<KVStoreListResponse>;
fn kvstore_delete(input: Json<KVStoreDeleteRequest>) -> Json<KVStoreDeleteResponse>;
fn kvstore_deletebyprefix(input: Json<KVStoreDeleteByPrefixRequest>) -> Json<KVStoreDeleteByPrefixResponse>;
fn kvstore_getstorageused(input: Json<serde_json::Value>) -> Json<KVStoreGetStorageUsedResponse>;
}
/// Set stores a byte value with the given key.
///
/// Parameters:
/// - key: The storage key (max 256 bytes, UTF-8)
/// - value: The byte slice to store
///
/// Returns an error if the storage limit would be exceeded or the operation fails.
///
/// # Arguments
/// * `key` - String parameter.
/// * `value` - Vec<u8> parameter.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn set(key: &str, value: Vec<u8>) -> Result<(), Error> {
let response = unsafe {
kvstore_set(Json(KVStoreSetRequest {
key: key.to_owned(),
value: value,
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(())
}
/// SetWithTTL stores a byte value with the given key and a time-to-live.
///
/// After ttlSeconds, the key is treated as non-existent and will be
/// cleaned up lazily. ttlSeconds must be greater than 0.
///
/// Parameters:
/// - key: The storage key (max 256 bytes, UTF-8)
/// - value: The byte slice to store
/// - ttlSeconds: Time-to-live in seconds (must be > 0)
///
/// Returns an error if the storage limit would be exceeded or the operation fails.
///
/// # Arguments
/// * `key` - String parameter.
/// * `value` - Vec<u8> parameter.
/// * `ttl_seconds` - i64 parameter.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn set_with_ttl(key: &str, value: Vec<u8>, ttl_seconds: i64) -> Result<(), Error> {
let response = unsafe {
kvstore_setwithttl(Json(KVStoreSetWithTTLRequest {
key: key.to_owned(),
value: value,
ttl_seconds: ttl_seconds,
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(())
}
/// Get retrieves a byte value from storage.
///
/// Parameters:
/// - key: The storage key
///
/// Returns the value and whether the key exists.
///
/// # Arguments
/// * `key` - String parameter.
///
/// # Returns
/// `Some(value)` if found, `None` otherwise.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn get(key: &str) -> Result<Option<Vec<u8>>, Error> {
let response = unsafe {
kvstore_get(Json(KVStoreGetRequest {
key: key.to_owned(),
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
if response.0.exists {
Ok(Some(response.0.value))
} else {
Ok(None)
}
}
/// GetMany retrieves multiple values in a single call.
///
/// Parameters:
/// - keys: The storage keys to retrieve
///
/// Returns a map of key to value for keys that exist and have not expired.
/// Missing or expired keys are omitted from the result.
///
/// # Arguments
/// * `keys` - Vec<String> parameter.
///
/// # Returns
/// The values value.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn get_many(keys: Vec<String>) -> Result<std::collections::HashMap<String, Vec<u8>>, Error> {
let response = unsafe {
kvstore_getmany(Json(KVStoreGetManyRequest {
keys: keys,
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(response.0.values)
}
/// Has checks if a key exists in storage.
///
/// Parameters:
/// - key: The storage key
///
/// Returns true if the key exists.
///
/// # Arguments
/// * `key` - String parameter.
///
/// # Returns
/// The exists value.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn has(key: &str) -> Result<bool, Error> {
let response = unsafe {
kvstore_has(Json(KVStoreHasRequest {
key: key.to_owned(),
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(response.0.exists)
}
/// List returns all keys matching the given prefix.
///
/// Parameters:
/// - prefix: Key prefix to filter by (empty string returns all keys)
///
/// Returns a slice of matching keys.
///
/// # Arguments
/// * `prefix` - String parameter.
///
/// # Returns
/// The keys value.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn list(prefix: &str) -> Result<Vec<String>, Error> {
let response = unsafe {
kvstore_list(Json(KVStoreListRequest {
prefix: prefix.to_owned(),
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(response.0.keys)
}
/// Delete removes a value from storage.
///
/// Parameters:
/// - key: The storage key
///
/// Returns an error if the operation fails. Does not return an error if the key doesn't exist.
///
/// # Arguments
/// * `key` - String parameter.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn delete(key: &str) -> Result<(), Error> {
let response = unsafe {
kvstore_delete(Json(KVStoreDeleteRequest {
key: key.to_owned(),
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(())
}
/// DeleteByPrefix removes all keys matching the given prefix.
///
/// Parameters:
/// - prefix: Key prefix to match (must not be empty)
///
/// Returns the number of keys deleted. Includes expired keys.
///
/// # Arguments
/// * `prefix` - String parameter.
///
/// # Returns
/// The deleted_count value.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn delete_by_prefix(prefix: &str) -> Result<i64, Error> {
let response = unsafe {
kvstore_deletebyprefix(Json(KVStoreDeleteByPrefixRequest {
prefix: prefix.to_owned(),
}))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(response.0.deleted_count)
}
/// GetStorageUsed returns the total storage used by this plugin in bytes.
///
/// # Returns
/// The bytes value.
///
/// # Errors
/// Returns an error if the host function call fails.
pub fn get_storage_used() -> Result<i64, Error> {
let response = unsafe {
kvstore_getstorageused(Json(serde_json::json!({})))?
};
if let Some(err) = response.0.error {
return Err(Error::msg(err));
}
Ok(response.0.bytes)
}