SeanOMik
/
lyra-engine
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

resource: asyncronous loading of resources

This commit is contained in:
SeanOMik 2024-03-09 00:25:13 -05:00
parent 1d7d13eb7b
commit dead32dbab
Signed by: SeanOMik
GPG Key ID: FEC9E2FC15235964
11 changed files with 524 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1839,6 +1839,7 @@ name = "lyra-resource"
version = "0.0.1"
dependencies = [
"anyhow",
"async-std",
"base64 0.21.5",
"crossbeam",
"glam",

3
lyra-resource/Cargo.toml
View File

@ -24,4 +24,5 @@ percent-encoding = "2.3.0"
thiserror = "1.0.48"
tracing = "0.1.37"
uuid = { version = "1.4.1", features = ["v4"] }
instant = "0.1"
instant = "0.1"
async-std = "1.12.0"

166
lyra-resource/src/gltf/loader.rs
View File

@ -1,12 +1,12 @@
use std::{sync::Arc, path::PathBuf};
use std::{ffi::OsStr, path::{Path, PathBuf}, sync::Arc};
use glam::{Quat, Vec3};
use instant::Instant;
use lyra_math::Transform;
use thiserror::Error;
use crate::{gltf::GltfScene, util, LoaderError, ResHandle, ResourceLoader, ResourceManager};
use super::{GltfNode, Material, Mesh, MeshIndices, MeshVertexAttribute, VertexAttributeData};
use crate::{gltf::GltfScene, loader::{LoaderError, PinedBoxLoaderFuture, ResourceLoader}, util, ResHandle, ResourceData, ResourceManager, ResourceStorage};
use super::{Gltf, GltfNode, Material, Mesh, MeshIndices, MeshVertexAttribute, VertexAttributeData};
use tracing::debug;
@ -32,7 +32,7 @@ impl From<ModelLoaderError> for LoaderError {
#[allow(dead_code)]
pub(crate) struct GltfLoadContext<'a> {
pub resource_manager: &'a mut ResourceManager,
pub resource_manager: ResourceManager,
pub gltf: &'a gltf::Gltf,
/// Path to the gltf
pub gltf_path: &'a str,
@ -125,7 +125,7 @@ impl ModelLoader {
new_mesh.material = Some(mat.clone());
}
let handle = ResHandle::with_data("", new_mesh);
let handle = ResHandle::new_ready(None, new_mesh);
ctx.resource_manager.store_uuid(handle.clone());
node.mesh = Some(handle);
}
@ -137,6 +137,21 @@ impl ModelLoader {
node
}
fn extensions() -> &'static [&'static str] {
&[
"gltf", "glb"
]
}
fn does_support_file(path: &str) -> bool {
match Path::new(path).extension().and_then(OsStr::to_str) {
Some(ext) => {
Self::extensions().contains(&ext)
},
_ => false,
}
}
}
impl ResourceLoader for ModelLoader {
@ -150,82 +165,92 @@ impl ResourceLoader for ModelLoader {
&[]
}
fn load(&self, resource_manager: &mut ResourceManager, path: &str) -> Result<std::sync::Arc<dyn crate::ResourceStorage>, crate::LoaderError> {
// check if the file is supported by this loader
if !self.does_support_file(path) {
return Err(LoaderError::UnsupportedExtension(path.to_string()));
}
fn load(&self, resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture {
// cant use &str across async
let path = path.to_string();
let mut parent_path = PathBuf::from(path);
parent_path.pop();
let parent_path = parent_path.display().to_string();
let gltf = gltf::Gltf::open(path)?;
let mut use_bin = false;
let buffers: Vec<Vec<u8>> = gltf.buffers().flat_map(|b| match b.source() {
gltf::buffer::Source::Bin => {
use_bin = true;
gltf.blob.as_deref().map(|v| v.to_vec())
.ok_or(ModelLoaderError::MissingBin(path.to_string()))
},
gltf::buffer::Source::Uri(uri) => util::gltf_read_buffer_uri(&parent_path, uri)
.map_err(ModelLoaderError::UriDecodingError),
}).collect();
let mut gltf_out = super::Gltf::default();
let mut context = GltfLoadContext {
resource_manager,
gltf: &gltf,
gltf_path: path,
gltf_parent_path: &parent_path,
buffers: &buffers,
};
let start_inst = Instant::now();
let materials: Vec<ResHandle<Material>> = gltf.materials()
.map(|mat| ResHandle::with_data("", Material::from_gltf(&mut context, mat)))
.collect();
let mat_time = Instant::now() - start_inst;
debug!("Loaded {} materials in {}s", materials.len(), mat_time.as_secs_f32());
for (idx, scene) in gltf.scenes().enumerate() {
let start_inst = Instant::now();
let nodes: Vec<GltfNode> = scene.nodes()
.map(|node| ModelLoader::process_node(&mut context, &materials, node))
.collect();
let node_time = Instant::now() - start_inst;
debug!("Loaded {} nodes in the scene in {}s", nodes.len(), node_time.as_secs_f32());
for mesh in nodes.iter().map(|n| &n.mesh) {
if let Some(mesh) = mesh {
gltf_out.meshes.push(mesh.clone());
}
Box::pin(async move {
// check if the file is supported by this loader
if !Self::does_support_file(&path) {
return Err(LoaderError::UnsupportedExtension(path.to_string()));
}
let scene = GltfScene {
nodes,
};
let scene = ResHandle::with_data(&format!("{}:Scene{}", path, idx), scene);
gltf_out.scenes.push(scene);
}
let mut parent_path = PathBuf::from(&path);
parent_path.pop();
let parent_path = parent_path.display().to_string();
gltf_out.materials = materials;
let gltf = gltf::Gltf::open(&path)?;
let mut use_bin = false;
let buffers: Vec<Vec<u8>> = gltf.buffers().flat_map(|b| match b.source() {
gltf::buffer::Source::Bin => {
use_bin = true;
gltf.blob.as_deref().map(|v| v.to_vec())
.ok_or(ModelLoaderError::MissingBin(path.to_string()))
},
gltf::buffer::Source::Uri(uri) => util::gltf_read_buffer_uri(&parent_path, uri)
.map_err(ModelLoaderError::UriDecodingError),
}).collect();
let mut gltf_out = super::Gltf::default();
let mut context = GltfLoadContext {
resource_manager: resource_manager.clone(),
gltf: &gltf,
gltf_path: &path,
gltf_parent_path: &parent_path,
buffers: &buffers,
};
let start_inst = Instant::now();
let materials: Vec<ResHandle<Material>> = gltf.materials()
.map(|mat| ResHandle::new_ready(None, Material::from_gltf(&mut context, mat)))
.collect();
let mat_time = Instant::now() - start_inst;
debug!("Loaded {} materials in {}s", materials.len(), mat_time.as_secs_f32());
for (_idx, scene) in gltf.scenes().enumerate() {
let start_inst = Instant::now();
let nodes: Vec<GltfNode> = scene.nodes()
.map(|node| ModelLoader::process_node(&mut context, &materials, node))
.collect();
let node_time = Instant::now() - start_inst;
Ok(Arc::new(ResHandle::with_data(path, gltf_out)))
debug!("Loaded {} nodes in the scene in {}s", nodes.len(), node_time.as_secs_f32());
for mesh in nodes.iter().map(|n| &n.mesh) {
if let Some(mesh) = mesh {
gltf_out.meshes.push(mesh.clone());
}
}
let scene = GltfScene {
nodes,
};
let scene = ResHandle::new_ready(Some(path.as_str()), scene);
gltf_out.scenes.push(scene);
}
gltf_out.materials = materials;
Ok(Box::new(gltf_out) as Box<dyn ResourceData>)
})
}
#[allow(unused_variables)]
fn load_bytes(&self, resource_manager: &mut ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> Result<Arc<dyn crate::ResourceStorage>, LoaderError> {
fn load_bytes(&self, resource_manager: ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> PinedBoxLoaderFuture {
todo!()
}
fn create_erased_handle(&self) -> Arc<dyn ResourceStorage> {
Arc::from(ResHandle::<Gltf>::new_loading(None))
}
}
#[cfg(test)]
mod tests {
use crate::{gltf::Gltf, ResourceLoader};
use crate::tests::busy_wait_resource;
use super::*;
fn test_file_path(path: &str) -> String {
@ -238,10 +263,9 @@ mod tests {
fn test_loading() {
let path = test_file_path("texture-embedded.gltf");
let mut manager = ResourceManager::new();
let loader = ModelLoader::default();
let gltf = loader.load(&mut manager, &path).unwrap();
let gltf = Arc::downcast::<ResHandle<Gltf>>(gltf.as_arc_any()).unwrap();
let manager = ResourceManager::new();
let gltf = manager.request::<Gltf>(&path).unwrap();
busy_wait_resource(&gltf, 15.0);
let gltf = gltf.data_ref().unwrap();
assert_eq!(gltf.scenes.len(), 1);

2
lyra-resource/src/gltf/material.rs
View File

@ -271,7 +271,7 @@ impl Material {
wrap_w: WrappingMode::ClampToEdge,
};
let handler = ResHandle::with_data("", Texture {
let handler = ResHandle::new_ready(None, Texture {
image: tex_img,
sampler: Some(samp),
});

9
lyra-resource/src/lib.rs
View File

@ -1,18 +1,17 @@
pub mod resource_manager;
mod resource_manager;
pub use resource_manager::*;
pub mod resource;
mod resource;
pub use resource::*;
pub mod texture;
mod texture;
pub use texture::*;
pub mod loader;
pub use loader::*;
pub mod gltf;
pub mod world_ext;
mod world_ext;
pub use world_ext::*;
pub(crate) mod util;

116
lyra-resource/src/loader/image.rs
View File

@ -1,11 +1,12 @@
use std::{fs::File, sync::Arc, io::Read};
use std::{ffi::OsStr, path::Path, sync::Arc};
use async_std::io::ReadExt;
use image::ImageError;
use tracing::{debug, trace};
use tracing::trace;
use crate::{resource::ResHandle, resource_manager::ResourceStorage, Image, ResourceManager};
use crate::{Image, ResHandle, ResourceData, ResourceManager};
use super::{LoaderError, ResourceLoader};
use super::{LoaderError, PinedBoxLoaderFuture, ResourceLoader};
impl From<ImageError> for LoaderError {
fn from(value: ImageError) -> Self {
@ -17,8 +18,8 @@ impl From<ImageError> for LoaderError {
#[derive(Default)]
pub struct ImageLoader;
impl ResourceLoader for ImageLoader {
fn extensions(&self) -> &[&str] {
impl ImageLoader {
fn extensions() -> &'static [&'static str] {
&[
// the extensions of these are the names of the formats
"bmp", "dds", "gif", "ico", "jpeg", "jpg", "png", "qoi", "tga", "tiff", "webp",
@ -31,6 +32,21 @@ impl ResourceLoader for ImageLoader {
]
}
fn does_support_file(path: &str) -> bool {
match Path::new(path).extension().and_then(OsStr::to_str) {
Some(ext) => {
Self::extensions().contains(&ext)
},
_ => false,
}
}
}
impl ResourceLoader for ImageLoader {
fn extensions(&self) -> &[&str] {
Self::extensions()
}
fn mime_types(&self) -> &[&str] {
&[
"image/bmp", "image/vnd.ms-dds", "image/gif", "image/x-icon", "image/jpeg",
@ -43,48 +59,55 @@ impl ResourceLoader for ImageLoader {
]
}
fn load(&self, _resource_manager: &mut ResourceManager, path: &str) -> Result<Arc<dyn ResourceStorage>, LoaderError> {
// check if the file is supported by this loader
if !self.does_support_file(path) {
return Err(LoaderError::UnsupportedExtension(path.to_string()));
}
fn load(&self, _resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture {
let path = path.to_string();
Box::pin(async move {
// check if the file is supported by this loader
if !Self::does_support_file(&path) {
return Err(LoaderError::UnsupportedExtension(path.to_string()));
}
// read file bytes
let mut file = File::open(path)?;
let mut buf = vec![];
file.read_to_end(&mut buf)?;
// read file bytes
let mut file = async_std::fs::File::open(path).await?;
let mut buf = vec![];
file.read_to_end(&mut buf).await?;
// load the image and construct Resource<Texture>
let image = image::load_from_memory(&buf)
.map_err(|e| match e {
ImageError::IoError(e) => LoaderError::IoError(e),
_ => LoaderError::DecodingError(e.into()),
})?;
let image = Image::from(image);
let res = ResHandle::with_data(path, image);
Ok(Arc::new(res))
// load the image and construct Resource<Texture>
let image = image::load_from_memory(&buf)
.map_err(|e| match e {
ImageError::IoError(e) => LoaderError::IoError(e),
_ => LoaderError::DecodingError(e.into()),
})?;
let image = Image::from(image);
let image = Box::new(image) as Box<dyn ResourceData>;
Ok(image)
})
}
fn load_bytes(&self, _resource_manager: &mut ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> Result<Arc<dyn ResourceStorage>, LoaderError> {
trace!("Loading {} bytes as an image", length);
let image = image::load_from_memory(&bytes[offset..(length-offset)])
.map_err(|e| match e {
ImageError::IoError(e) => LoaderError::IoError(e),
_ => LoaderError::DecodingError(e.into()),
})?;
let image = Image::from(image);
let res = ResHandle::with_data(&uuid::Uuid::new_v4().to_string(), image);
debug!("Finished loading image of {} bytes", length);
fn load_bytes(&self, _resource_manager: ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> PinedBoxLoaderFuture {
Box::pin(async move {
trace!("Loading {} bytes as an image", length);
let image = image::load_from_memory(&bytes[offset..(length-offset)])
.map_err(|e| match e {
ImageError::IoError(e) => LoaderError::IoError(e),
_ => LoaderError::DecodingError(e.into()),
})?;
let image = Image::from(image);
Ok(Box::new(image) as Box<dyn ResourceData>)
})
}
Ok(Arc::new(res))
fn create_erased_handle(&self) -> Arc<dyn crate::ResourceStorage> {
Arc::from(ResHandle::<Image>::new_loading(None))
}
}
#[cfg(test)]
mod tests {
use async_std::task;
use super::*;
fn get_image(path: &str) -> String {
@ -102,15 +125,24 @@ mod tests {
/// Tests loading an image
#[test]
fn image_load() {
let mut manager = ResourceManager::new();
let manager = ResourceManager::new();
let loader = ImageLoader::default();
loader.load(&mut manager, &get_image("squiggles.png")).unwrap();
task::block_on(async move {
let r = loader.load(manager, &get_image("squiggles.png")).await.unwrap();
let a = r.as_ref();
a.as_any().downcast_ref::<Image>().unwrap();
});
}
#[test]
fn image_load_unsupported() {
let mut manager = ResourceManager::new();
let manager = ResourceManager::new();
let loader = ImageLoader::default();
assert!(loader.load(&mut manager, &get_image("squiggles.gltf")).is_err());
task::block_on(async move {
// this file doesn't exist and is also not supported
assert!(loader.load(manager, &get_image("squiggles.jfeh")).await.is_err())
});
}
}

27
lyra-resource/src/loader/mod.rs
View File

@ -1,10 +1,11 @@
pub mod image;
mod image;
pub use image::*;
use std::{io, sync::Arc, path::Path, ffi::OsStr};
use std::{ffi::OsStr, future::Future, io, path::Path, pin::Pin, sync::Arc};
use thiserror::Error;
use crate::{resource_manager::ResourceStorage, ResourceManager};
use crate::{resource_manager::ResourceStorage, ResourceData, ResourceManager};
#[derive(Error, Debug)]
pub enum LoaderError {
@ -28,7 +29,9 @@ impl From<io::Error> for LoaderError {
}
}
pub trait ResourceLoader {
pub type PinedBoxLoaderFuture = Pin<Box<dyn Future<Output = Result<Box<dyn ResourceData>, LoaderError>> + Send>>;
pub trait ResourceLoader: Send + Sync {
/// Returns the extensions that this loader supports. Does not include the `.`
fn extensions(&self) -> &[&str];
/// Returns the mime types that this loader supports.
@ -50,9 +53,21 @@ pub trait ResourceLoader {
}
/// Load a resource from a path.
fn load(&self, resource_manager: &mut ResourceManager, path: &str) -> Result<Arc<dyn ResourceStorage>, LoaderError>;
fn load(&self, resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture;
/// Load a resource from bytes.
fn load_bytes(&self, resource_manager: &mut ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> Result<Arc<dyn ResourceStorage>, LoaderError>;
fn load_bytes(&self, resource_manager: ResourceManager, bytes: Vec<u8>, offset: usize, length: usize) -> PinedBoxLoaderFuture;
/// Creates and returns a `ResHandle` in a loading state that stores the type that this
/// loader returns.
///
/// This is very simple to implement, you can use this as a template:
/// ```nobuild
/// fn create_erased_handle(&self) -> Arc<dyn crate::ResourceStorage> {
/// // Change the type of the reshandle to match the loader.
/// Arc::from(ResHandle::<Image>::new_loading(None))
/// }
/// ```
fn create_erased_handle(&self) -> Arc<dyn ResourceStorage>;
}

110
lyra-resource/src/resource.rs
View File

@ -1,37 +1,64 @@
use std::{any::Any, sync::{Arc, RwLock}};
use std::{any::{Any, TypeId}, marker::PhantomData, sync::{Arc, RwLock}};
use lyra_ecs::Component;
use crate::lyra_engine;
use crate::{loader::LoaderError, lyra_engine};
use uuid::Uuid;
use crate::ResourceStorage;
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
/// A trait that that each resource type should implement.
pub trait ResourceData: Send + Sync + Any + 'static {
fn as_any(&self) -> &dyn Any;
fn as_any_mut(&mut self) -> &mut dyn Any;
fn type_id(&self) -> TypeId;
}
impl<T: Send + Sync + Any + 'static> ResourceData for T {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn type_id(&self) -> TypeId {
TypeId::of::<T>()
}
}
pub enum ResourceState {
Loading,
Ready,
Error(Arc<LoaderError>),
Ready(Box<dyn ResourceData>),
}
pub struct ResourceDataRef<'a, T> {
guard: std::sync::RwLockReadGuard<'a, Resource<T>>,
}
impl<'a, T> std::ops::Deref for ResourceDataRef<'a, T> {
impl<'a, T: 'static> std::ops::Deref for ResourceDataRef<'a, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
// safety: this struct must only be created if the resource is loaded
self.guard.data.as_ref().unwrap()
match &self.guard.state {
ResourceState::Ready(d) => {
// for some reason, if I didn't use `.as_ref`, the downcast would fail.
let d = d.as_ref().as_any();
d.downcast_ref::<T>().unwrap()
},
_ => unreachable!() // ResHandler::data_ref shouldn't allow this to run
}
}
}
pub struct Resource<T> {
path: String,
pub(crate) data: Option<T>,
pub(crate) version: usize,
pub(crate) state: ResourceState,
uuid: Uuid,
path: Option<String>,
pub(crate) is_watched: bool,
_marker: PhantomData<T>,
}
/// A handle to a resource.
@ -51,16 +78,31 @@ impl<T> Clone for ResHandle<T> {
}
}
impl<T> ResHandle<T> {
/// Create the resource with data, its assumed the state is `Ready`
pub fn with_data(path: &str, data: T) -> Self {
impl<T: ResourceData> ResHandle<T> {
pub fn new_loading(path: Option<&str>) -> Self {
let res_version = Resource {
path: path.to_string(),
data: Some(data),
version: 0,
state: ResourceState::Ready,
path: path.map(str::to_string),
state: ResourceState::Loading,
uuid: Uuid::new_v4(),
is_watched: false,
_marker: PhantomData::<T>
};
Self {
data: Arc::new(RwLock::new(res_version)),
}
}
/// Create the resource with data, its assumed the state is `Ready`
pub fn new_ready(path: Option<&str>, data: T) -> Self {
let res_version = Resource {
version: 0,
path: path.map(str::to_string),
state: ResourceState::Ready(Box::new(data)),
uuid: Uuid::new_v4(),
is_watched: false,
_marker: PhantomData::<T>
};
Self {
@ -77,20 +119,14 @@ impl<T> ResHandle<T> {
/// Returns a boolean indicating if this resource is loaded
pub fn is_loaded(&self) -> bool {
let d = self.data.read().expect("Resource mutex was poisoned!");
d.state == ResourceState::Ready
matches!(d.state, ResourceState::Ready(_))
}
/// Returns the current state of the resource.
pub fn state(&self) -> ResourceState {
/* pub fn state(&self) -> &ResourceState {
let d = self.data.read().expect("Resource mutex was poisoned!");
d.state
}
/// Returns the path that the resource was loaded from.
pub fn path(&self) -> String {
let d = self.data.read().expect("Resource mutex was poisoned!");
d.path.to_string()
}
&d.state
} */
/// Returns the uuid of the resource.
pub fn uuid(&self) -> Uuid {
@ -98,6 +134,11 @@ impl<T> ResHandle<T> {
d.uuid
}
pub fn path(&self) -> Option<String> {
let d = self.data.read().expect("Resource mutex was poisoned!");
d.path.clone()
}
/// Retrieves the current version of the resource. This gets incremented when the resource
/// is reloaded.
pub fn version(&self) -> usize {
@ -140,22 +181,18 @@ impl<T: Send + Sync + 'static> ResourceStorage for ResHandle<T> {
w.is_watched = watched;
}
fn path(&self) -> String {
self.path()
}
fn version(&self) -> usize {
self.version()
}
fn state(&self) -> ResourceState {
self.state()
}
fn uuid(&self) -> Uuid {
self.uuid()
}
fn path(&self) -> Option<String> {
self.path()
}
fn is_watched(&self) -> bool {
self.is_watched()
}
@ -163,4 +200,11 @@ impl<T: Send + Sync + 'static> ResourceStorage for ResHandle<T> {
fn is_loaded(&self) -> bool {
self.is_loaded()
}
fn set_state(&self, new: ResourceState) {
let mut d = self.data.write().expect("Resource mutex was poisoned!");
d.state = new;
}
}

353
lyra-resource/src/resource_manager.rs
View File

@ -1,12 +1,13 @@
use std::{sync::{Arc, RwLock}, collections::HashMap, any::Any, path::Path, time::Duration};
use std::{any::Any, collections::HashMap, path::Path, sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, time::Duration};
use async_std::task;
use crossbeam::channel::Receiver;
use notify::{Watcher, RecommendedWatcher};
use notify_debouncer_full::{DebouncedEvent, FileIdMap};
use thiserror::Error;
use uuid::Uuid;
use crate::{gltf::ModelLoader, loader::{image::ImageLoader, LoaderError, ResourceLoader}, resource::ResHandle, ResourceState};
use crate::{gltf::ModelLoader, loader::{ImageLoader, LoaderError, ResourceLoader}, resource::ResHandle, ResourceState};
/// A trait for type erased storage of a resource.
/// Implemented for [`ResHandle<T>`]
@ -19,24 +20,30 @@ pub trait ResourceStorage: Send + Sync + Any + 'static {
/// This is used internally.
fn set_watched(&self, watched: bool);
fn path(&self) -> String;
fn version(&self) -> usize;
fn state(&self) -> ResourceState;
fn uuid(&self) -> Uuid;
fn path(&self) -> Option<String>;
fn is_watched(&self) -> bool;
fn is_loaded(&self) -> bool;
fn set_state(&self, new: ResourceState);
}
#[derive(Error, Debug)]
pub enum RequestError {
#[error("{0}")]
Loader(LoaderError),
#[error("The file extension is unsupported: '{0}'")]
UnsupportedFileExtension(String),
#[error("The mimetype is unsupported: '{0}'")]
UnsupportedMime(String),
#[error("The identifier is not found: '{0}'")]
IdentNotFound(String),
#[error("The resource was not loaded from a path so cannot be reloaded")]
NoReloadPath
}
impl From<LoaderError> for RequestError {
@ -45,51 +52,79 @@ impl From<LoaderError> for RequestError {
}
}
/// A struct that stores all Manager data. This is requried for sending
//struct ManagerStorage
/// A struct that
/// A struct that stores some things used for watching resources.
pub struct ResourceWatcher {
debouncer: Arc<RwLock<notify_debouncer_full::Debouncer<RecommendedWatcher, FileIdMap>>>,
events_recv: Receiver<Result<Vec<DebouncedEvent>, Vec<notify::Error>>>,
}
pub struct ResourceManager {
/// The state of the ResourceManager
pub struct ResourceManagerState {
resources: HashMap<String, Arc<dyn ResourceStorage>>,
uuid_resources: HashMap<Uuid, Arc<dyn ResourceStorage>>,
loaders: Vec<Arc<dyn ResourceLoader>>,
watchers: HashMap<String, ResourceWatcher>,
}
/// The ResourceManager
///
/// This exists since we need the manager to be `Send + Sync`.
#[derive(Clone)]
pub struct ResourceManager {
inner: Arc<RwLock<ResourceManagerState>>,
}
impl Default for ResourceManager {
fn default() -> Self {
Self::new()
Self {
inner: Arc::new(RwLock::new(
ResourceManagerState {
resources: HashMap::new(),
loaders: vec![ Arc::new(ImageLoader), Arc::new(ModelLoader) ],
watchers: HashMap::new(),
}
))
}
}
}
impl ResourceManager {
pub fn new() -> Self {
Self {
resources: HashMap::new(),
uuid_resources: HashMap::new(),
loaders: vec![ Arc::new(ImageLoader), Arc::new(ModelLoader) ],
watchers: HashMap::new(),
}
Self::default()
}
/// Retrieves a non-mutable guard of the manager's state.
pub fn state(&self) -> RwLockReadGuard<ResourceManagerState> {
self.inner.read().unwrap()
}
/// Retrieves a mutable guard of the manager's state.
pub fn state_mut(&self) -> RwLockWriteGuard<ResourceManagerState> {
self.inner.write().unwrap()
}
/// Registers a loader to the manager.
pub fn register_loader<L>(&mut self)
pub fn register_loader<L>(&self)
where
L: ResourceLoader + Default + 'static
{
self.loaders.push(Arc::new(L::default()));
let mut state = self.state_mut();
state.loaders.push(Arc::new(L::default()));
}
pub fn request<T>(&mut self, path: &str) -> Result<ResHandle<T>, RequestError>
/// Request a resource at `path`.
///
/// When a resource for a path is requested for the first time, the resource will be loaded
/// and cached. In the future, the cached version will be returned. There is only ever one copy
/// of the resource's data in memory at a time.
///
/// Loading resources is done asynchronously on a task spawned by `async-std`. You can use the
/// handle to check if the resource is loaded.
pub fn request<T>(&self, path: &str) -> Result<ResHandle<T>, RequestError>
where
T: Send + Sync + Any + 'static
{
match self.resources.get(&path.to_string()) {
let mut state = self.state_mut();
match state.resources.get(&path.to_string()) {
Some(res) => {
let res = res.clone().as_arc_any();
let res: Arc<ResHandle<T>> = res.downcast::<ResHandle<T>>().expect("Failure to downcast resource");
@ -98,22 +133,33 @@ impl ResourceManager {
Ok(res)
},
None => {
if let Some(loader) = self.loaders.iter()
if let Some(loader) = state.loaders.iter()
.find(|l| l.does_support_file(path)) {
// Load the resource and store it
let loader = Arc::clone(loader); // stop borrowing from self
let res = loader.load(self, path)?;
let res: Arc<dyn ResourceStorage> = Arc::from(res);
self.resources.insert(path.to_string(), res.clone());
let res = loader.load(self.clone(), path);
let handle = ResHandle::<T>::new_loading(Some(path));
let thand = handle.clone();
task::spawn(async move {
match res.await {
Ok(data) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Ready(data);
}
Err(err) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Error(Arc::new(err));
}
}
});
// cast Arc<dyn ResourceStorage> to Arc<Resource<T>
let res = res.as_arc_any();
let res = res.downcast::<ResHandle<T>>()
.expect("Failure to downcast resource! Does the loader return an `Arc<Resource<T>>`?");
let res = ResHandle::<T>::clone(&res);
let res: Arc<dyn ResourceStorage> = Arc::from(handle.clone());
state.resources.insert(path.to_string(), res);
Ok(res)
Ok(handle)
} else {
Err(RequestError::UnsupportedFileExtension(path.to_string()))
}
@ -121,28 +167,42 @@ impl ResourceManager {
}
}
/// Request a resource without downcasting to a ResHandle<T>.
/// Request a resource without downcasting to a `ResHandle<T>`.
/// Whenever you're ready to downcast, you can do so like this:
/// ```compile_fail
/// let arc_any = res_arc.as_arc_any();
/// let res: Arc<ResHandle<T>> = res.downcast::<ResHandle<T>>().expect("Failure to downcast resource");
/// ```
pub fn request_raw(&mut self, path: &str) -> Result<Arc<dyn ResourceStorage>, RequestError> {
match self.resources.get(&path.to_string()) {
pub fn request_raw(&self, path: &str) -> Result<Arc<dyn ResourceStorage>, RequestError> {
let inner = self.inner.write().unwrap();
match inner.resources.get(&path.to_string()) {
Some(res) => {
Ok(res.clone())
},
None => {
if let Some(loader) = self.loaders.iter()
if let Some(loader) = inner.loaders.iter()
.find(|l| l.does_support_file(path)) {
// Load the resource and store it
let loader = Arc::clone(loader); // stop borrowing from self
let res = loader.load(self, path)?;
let res: Arc<dyn ResourceStorage> = Arc::from(res);
self.resources.insert(path.to_string(), res.clone());
let res = loader.load(self.clone(), path);
Ok(res)
let handle = loader.create_erased_handle();
//let handle = ResHandle::<T>::new_loading();
let thand = handle.clone();
task::spawn(async move {
match res.await {
Ok(data) => {
thand.set_state(ResourceState::Ready(data));
}
Err(err) => {
thand.set_state(ResourceState::Error(Arc::new(err)));
}
}
});
Ok(handle)
} else {
Err(RequestError::UnsupportedFileExtension(path.to_string()))
}
@ -154,16 +214,18 @@ impl ResourceManager {
///
/// The resource cannot be requested with [`ResourceManager::request`], it can only be
/// retrieved with [`ResourceManager::request_uuid`].
pub fn store_uuid<T: Send + Sync + 'static>(&mut self, res: ResHandle<T>) {
self.uuid_resources.insert(res.uuid(), Arc::new(res));
pub fn store_uuid<T: Send + Sync + 'static>(&self, res: ResHandle<T>) {
let mut state = self.state_mut();
state.resources.insert(res.uuid().to_string(), Arc::new(res));
}
/// Request a resource via its uuid.
///
/// Returns `None` if the resource was not found. The resource must of had been
/// stored with [`ResourceManager::request`] to return `Some`.
pub fn request_uuid<T: Send + Sync + 'static>(&mut self, uuid: &Uuid) -> Option<ResHandle<T>> {
match self.uuid_resources.get(uuid) {
pub fn request_uuid<T: Send + Sync + 'static>(&self, uuid: &Uuid) -> Option<ResHandle<T>> {
let state = self.state();
match state.resources.get(&uuid.to_string()) {
Some(res) => {
let res = res.clone().as_arc_any();
let res: Arc<ResHandle<T>> = res.downcast::<ResHandle<T>>().expect("Failure to downcast resource");
@ -182,36 +244,47 @@ impl ResourceManager {
/// * `bytes` - The bytes to store.
///
/// Returns: The `Arc` to the now stored resource
pub fn load_bytes<T>(&mut self, ident: &str, mime_type: &str, bytes: Vec<u8>, offset: usize, length: usize) -> Result<ResHandle<T>, RequestError>
pub fn load_bytes<T>(&self, ident: &str, mime_type: &str, bytes: Vec<u8>, offset: usize, length: usize) -> Result<ResHandle<T>, RequestError>
where
T: Send + Sync + Any + 'static
{
if let Some(loader) = self.loaders.iter()
let mut state = self.state_mut();
if let Some(loader) = state.loaders.iter()
.find(|l| l.does_support_mime(mime_type)) {
let loader = loader.clone();
let res = loader.load_bytes(self, bytes, offset, length)?;
let res: Arc<dyn ResourceStorage> = Arc::from(res);
self.resources.insert(ident.to_string(), res.clone());
// code here...
let res = loader.load_bytes(self.clone(), bytes, offset, length);
// cast Arc<dyn ResourceStorage> to Arc<Resource<T>
let res = res.as_arc_any();
let res = res.downcast::<ResHandle<T>>()
.expect("Failure to downcast resource! Does the loader return an `Arc<Resource<T>>`?");
let res = ResHandle::<T>::clone(&res);
let handle = ResHandle::<T>::new_loading(None);
let thand = handle.clone();
task::spawn(async move {
match res.await {
Ok(data) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Ready(data);
}
Err(err) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Error(Arc::new(err));
}
}
});
Ok(res)
let res: Arc<dyn ResourceStorage> = Arc::from(handle.clone());
state.resources.insert(ident.to_string(), res);
Ok(handle)
} else {
Err(RequestError::UnsupportedMime(mime_type.to_string()))
}
}
/// Requests bytes from the manager.
pub fn request_loaded_bytes<T>(&mut self, ident: &str) -> Result<Arc<ResHandle<T>>, RequestError>
pub fn request_loaded_bytes<T>(&self, ident: &str) -> Result<Arc<ResHandle<T>>, RequestError>
where
T: Send + Sync + Any + 'static
{
match self.resources.get(&ident.to_string()) {
let state = self.state();
match state.resources.get(&ident.to_string()) {
Some(res) => {
let res = res.clone().as_arc_any();
let res = res.downcast::<ResHandle<T>>().expect("Failure to downcast resource");
@ -225,7 +298,7 @@ impl ResourceManager {
}
/// Start watching a path for changes. Returns a mspc channel that will send events
pub fn watch(&mut self, path: &str, recursive: bool) -> notify::Result<Receiver<Result<Vec<DebouncedEvent>, Vec<notify::Error>>>> {
pub fn watch(&self, path: &str, recursive: bool) -> notify::Result<Receiver<Result<Vec<DebouncedEvent>, Vec<notify::Error>>>> {
let (send, recv) = crossbeam::channel::bounded(15);
let mut watcher = notify_debouncer_full::new_debouncer(Duration::from_millis(1000), None, send)?;
@ -241,9 +314,10 @@ impl ResourceManager {
events_recv: recv.clone(),
};
self.watchers.insert(path.to_string(), watcher);
let mut state = self.state_mut();
state.watchers.insert(path.to_string(), watcher);
let res = self.resources.get(&path.to_string())
let res = state.resources.get(&path.to_string())
.expect("The path that was watched has not been loaded as a resource yet");
res.set_watched(true);
@ -251,13 +325,14 @@ impl ResourceManager {
}
/// Stops watching a path
pub fn stop_watching(&mut self, path: &str) -> notify::Result<()> {
if let Some(watcher) = self.watchers.get(path) {
pub fn stop_watching(&self, path: &str) -> notify::Result<()> {
let state = self.state();
if let Some(watcher) = state.watchers.get(path) {
let mut watcher = watcher.debouncer.write().unwrap();
watcher.watcher().unwatch(Path::new(path))?;
// unwrap is safe since only loaded resources can be watched
let res = self.resources.get(&path.to_string()).unwrap();
let res = state.resources.get(&path.to_string()).unwrap();
res.set_watched(false);
}
@ -267,43 +342,50 @@ impl ResourceManager {
/// Returns a mspc receiver for watcher events of a specific path. The path must already
/// be watched with [`ResourceManager::watch`] for this to return `Some`.
pub fn watcher_event_recv(&self, path: &str) -> Option<Receiver<Result<Vec<DebouncedEvent>, Vec<notify::Error>>>> {
self.watchers.get(&path.to_string())
let state = self.state();
state.watchers.get(&path.to_string())
.map(|w| w.events_recv.clone())
}
/// Reloads a resource. The data inside the resource will be updated, the state may
pub fn reload<T>(&mut self, resource: ResHandle<T>) -> Result<(), RequestError>
/// Trigger a reload of a resource.
///
/// The version of the resource will be incremented by one.
///
/// > Note: Since reloading is done asynchronously, the reloaded data will not be immediately
/// accessible. Until the resource is reloaded, the previous data will stay inside of
/// the handle.
pub fn reload<T>(&self, resource: ResHandle<T>) -> Result<(), RequestError>
where
T: Send + Sync + Any + 'static
{
let path = resource.path();
if let Some(loader) = self.loaders.iter()
.find(|l| l.does_support_file(&path)) {
println!("got loader");
let loader = Arc::clone(loader); // stop borrowing from self
let loaded = loader.load(self, &path)?;
let loaded = loaded.as_arc_any();
let loaded = loaded.downcast::<ResHandle<T>>()
.unwrap();
let loaded = match Arc::try_unwrap(loaded) {
Ok(v) => v,
Err(_) => panic!("Seems impossible that this would happen, the resource was just loaded!"),
};
let loaded = loaded.data;
let loaded = match Arc::try_unwrap(loaded) {
Ok(v) => v,
Err(_) => panic!("Seems impossible that this would happen, the resource was just loaded!"),
};
let loaded = loaded.into_inner().unwrap();
let state = self.state();
let res_lock = &resource.data;
let path = resource.path()
.ok_or(RequestError::NoReloadPath)?;
if let Some(loader) = state.loaders.iter()
.find(|l| l.does_support_file(&path)) {
let loader = Arc::clone(loader); // stop borrowing from self
let res = loader.load(self.clone(), &path);
/* let res_lock = &resource.data;
let mut res_lock = res_lock.write().unwrap();
let version = res_lock.version;
res_lock.state = ResourceState::Loading;
drop(res_lock); */
res_lock.data = loaded.data;
res_lock.state = loaded.state;
res_lock.version = version + 1;
let thand = resource.clone();
task::spawn(async move {
match res.await {
Ok(data) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Ready(data);
d.version += 1;
}
Err(err) => {
let mut d = thand.data.write().unwrap();
d.state = ResourceState::Error(Arc::new(err));
}
}
});
}
Ok(())
@ -311,10 +393,12 @@ impl ResourceManager {
}
#[cfg(test)]
mod tests {
use std::io;
pub(crate) mod tests {
use std::{io, ops::Deref};
use crate::{Texture, ResourceState};
use instant::Instant;
use crate::{Image, ResourceData, Texture};
use super::*;
@ -324,37 +408,81 @@ mod tests {
format!("{manifest}/test_files/img/{path}")
}
pub(crate) fn busy_wait_resource<R: ResourceData>(handle: &ResHandle<R>, timeout: f32) {
let start = Instant::now();
while !handle.is_loaded() {
// loop until the image is loaded
let now = Instant::now();
let diff = now - start;
if diff.as_secs_f32() >= timeout {
panic!("Image never loaded");
}
}
}
pub(crate) fn busy_wait_resource_reload<R: ResourceData>(handle: &ResHandle<R>, timeout: f32) {
let version = handle.version();
let start = Instant::now();
while !handle.is_loaded() || handle.version() == version {
// loop until the image is loaded
let now = Instant::now();
let diff = now - start;
if diff.as_secs_f32() >= timeout {
panic!("Image never loaded");
}
}
}
#[test]
fn load_image() {
let mut man = ResourceManager::new();
let res = man.request::<Texture>(&get_image("squiggles.png")).unwrap();
assert_eq!(res.state(), ResourceState::Ready);
let img = res.data_ref();
img.unwrap();
let man = ResourceManager::new();
let res = man.request::<Image>(&get_image("squiggles.png")).unwrap();
assert!(!res.is_loaded());
busy_wait_resource(&res, 10.0);
// shouldn't panic because of the loop
res.data_ref().unwrap();
}
/// Ensures that only one copy of the same data was made
#[test]
fn ensure_single() {
let mut man = ResourceManager::new();
let man = ResourceManager::new();
let res = man.request::<Texture>(&get_image("squiggles.png")).unwrap();
assert_eq!(Arc::strong_count(&res.data), 2);
assert_eq!(Arc::strong_count(&res.data), 3);
let resagain = man.request::<Texture>(&get_image("squiggles.png")).unwrap();
assert_eq!(Arc::strong_count(&resagain.data), 3);
assert_eq!(Arc::strong_count(&resagain.data), 4);
}
/// Ensures that an error is returned when a file that doesn't exist is requested
#[test]
fn ensure_none() {
let mut man = ResourceManager::new();
let res = man.request::<Texture>(&get_image("squigglesfff.png"));
let err = res.err().unwrap();
let man = ResourceManager::new();
let res = man.request::<Texture>(&get_image("squigglesfff.png")).unwrap();
//let err = res.err().unwrap();
// 1 second should be enough to run into an error
std::thread::sleep(Duration::from_secs(1));
//busy_wait_resource(&res, 10.0);
let state = &res.data.read().unwrap().state;
assert!(
match err {
match state {
// make sure the error is NotFound
RequestError::Loader(LoaderError::IoError(e)) if e.kind() == io::ErrorKind::NotFound => true,
//RequestError::Loader(LoaderError::IoError(e)) if e.kind() == io::ErrorKind::NotFound => true,
ResourceState::Error(err) => {
match err.deref() {
LoaderError::IoError(e) if e.kind() == io::ErrorKind::NotFound => true,
_ => false,
}
},
_ => false
}
);
@ -362,17 +490,18 @@ mod tests {
#[test]
fn reload_image() {
let mut man = ResourceManager::new();
let man = ResourceManager::new();
let res = man.request::<Texture>(&get_image("squiggles.png")).unwrap();
assert_eq!(res.state(), ResourceState::Ready);
busy_wait_resource(&res, 10.0);
let img = res.data_ref();
img.unwrap();
println!("Path = {}", res.path());
man.reload(res.clone()).unwrap();
busy_wait_resource_reload(&res, 10.0);
assert_eq!(res.version(), 1);
man.reload(res.clone()).unwrap();
busy_wait_resource_reload(&res, 10.0);
assert_eq!(res.version(), 2);
}
@ -382,9 +511,9 @@ mod tests {
let image_path = get_image("squiggles_test.png");
std::fs::copy(orig_path, &image_path).unwrap();
let mut man = ResourceManager::new();
let man = ResourceManager::new();
let res = man.request::<Texture>(&image_path).unwrap();
assert_eq!(res.state(), ResourceState::Ready);
busy_wait_resource(&res, 10.0);
let img = res.data_ref();
img.unwrap();

12
lyra-resource/src/world_ext.rs
View File

@ -4,7 +4,7 @@ use crossbeam::channel::Receiver;
use lyra_ecs::World;
use notify_debouncer_full::DebouncedEvent;
use crate::{RequestError, ResHandle, ResourceLoader, ResourceManager};
use crate::{loader::ResourceLoader, RequestError, ResHandle, ResourceManager};
pub trait WorldAssetExt {
/// Register a resource loader with the resource manager.
@ -38,7 +38,7 @@ impl WorldAssetExt for World {
where
L: ResourceLoader + Default + 'static
{
let mut man = self.get_resource_or_default::<ResourceManager>();
let man = self.get_resource_or_default::<ResourceManager>();
man.register_loader::<L>();
}
@ -46,17 +46,17 @@ impl WorldAssetExt for World {
where
T: Send + Sync + Any + 'static
{
let mut man = self.get_resource_or_default::<ResourceManager>();
let man = self.get_resource_or_default::<ResourceManager>();
man.request(path)
}
fn watch_res(&mut self, path: &str, recursive: bool) -> notify::Result<Receiver<Result<Vec<DebouncedEvent>, Vec<notify::Error>>>> {
let mut man = self.get_resource_or_default::<ResourceManager>();
let man = self.get_resource_or_default::<ResourceManager>();
man.watch(path, recursive)
}
fn stop_watching_res(&mut self, path: &str) -> notify::Result<()> {
let mut man = self.get_resource_or_default::<ResourceManager>();
let man = self.get_resource_or_default::<ResourceManager>();
man.stop_watching(path)
}
@ -69,7 +69,7 @@ impl WorldAssetExt for World {
where
T: Send + Sync + Any + 'static
{
let mut man = self.get_resource_or_default::<ResourceManager>();
let man = self.get_resource_or_default::<ResourceManager>();
man.reload(resource)
}
}

4
lyra-scripting/src/lua/loader.rs
View File

@ -23,7 +23,7 @@ impl ResourceLoader for LuaLoader {
fn load(&self, _resource_manager: &mut lyra_resource::ResourceManager, path: &str) -> Result<std::sync::Arc<dyn lyra_resource::ResourceStorage>, lyra_resource::LoaderError> {
let bytes = std::fs::read(path)?;
let s = ResHandle::with_data(path, LuaScript {
let s = ResHandle::new_ready(path, LuaScript {
bytes
});
@ -34,7 +34,7 @@ impl ResourceLoader for LuaLoader {
let end = offset + length;
let bytes = bytes[offset..end].to_vec();
let s = ResHandle::with_data("from bytes", LuaScript {
let s = ResHandle::new_ready("from bytes", LuaScript {
bytes
});