From dead32dbabe7ff67f8e45a928e49cd13e1397143 Mon Sep 17 00:00:00 2001 From: SeanOMik Date: Sat, 9 Mar 2024 00:25:13 -0500 Subject: [PATCH] resource: asyncronous loading of resources --- Cargo.lock | 1 + lyra-resource/Cargo.toml | 3 +- lyra-resource/src/gltf/loader.rs | 166 ++++++------ lyra-resource/src/gltf/material.rs | 2 +- lyra-resource/src/lib.rs | 9 +- lyra-resource/src/loader/image.rs | 116 ++++++--- lyra-resource/src/loader/mod.rs | 27 +- lyra-resource/src/resource.rs | 110 +++++--- lyra-resource/src/resource_manager.rs | 353 ++++++++++++++++++-------- lyra-resource/src/world_ext.rs | 12 +- lyra-scripting/src/lua/loader.rs | 4 +- 11 files changed, 524 insertions(+), 279 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7c50da..61318a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1839,6 +1839,7 @@ name = "lyra-resource" version = "0.0.1" dependencies = [ "anyhow", + "async-std", "base64 0.21.5", "crossbeam", "glam", diff --git a/lyra-resource/Cargo.toml b/lyra-resource/Cargo.toml index a79817c..64083b0 100644 --- a/lyra-resource/Cargo.toml +++ b/lyra-resource/Cargo.toml @@ -24,4 +24,5 @@ percent-encoding = "2.3.0" thiserror = "1.0.48" tracing = "0.1.37" uuid = { version = "1.4.1", features = ["v4"] } -instant = "0.1" \ No newline at end of file +instant = "0.1" +async-std = "1.12.0" diff --git a/lyra-resource/src/gltf/loader.rs b/lyra-resource/src/gltf/loader.rs index 4298756..665d101 100644 --- a/lyra-resource/src/gltf/loader.rs +++ b/lyra-resource/src/gltf/loader.rs @@ -1,12 +1,12 @@ -use std::{sync::Arc, path::PathBuf}; +use std::{ffi::OsStr, path::{Path, PathBuf}, sync::Arc}; use glam::{Quat, Vec3}; use instant::Instant; use lyra_math::Transform; use thiserror::Error; -use crate::{gltf::GltfScene, util, LoaderError, ResHandle, ResourceLoader, ResourceManager}; -use super::{GltfNode, Material, Mesh, MeshIndices, MeshVertexAttribute, VertexAttributeData}; +use crate::{gltf::GltfScene, loader::{LoaderError, PinedBoxLoaderFuture, ResourceLoader}, util, ResHandle, ResourceData, ResourceManager, ResourceStorage}; +use super::{Gltf, GltfNode, Material, Mesh, MeshIndices, MeshVertexAttribute, VertexAttributeData}; use tracing::debug; @@ -32,7 +32,7 @@ impl From for LoaderError { #[allow(dead_code)] pub(crate) struct GltfLoadContext<'a> { - pub resource_manager: &'a mut ResourceManager, + pub resource_manager: ResourceManager, pub gltf: &'a gltf::Gltf, /// Path to the gltf pub gltf_path: &'a str, @@ -125,7 +125,7 @@ impl ModelLoader { new_mesh.material = Some(mat.clone()); } - let handle = ResHandle::with_data("", new_mesh); + let handle = ResHandle::new_ready(None, new_mesh); ctx.resource_manager.store_uuid(handle.clone()); node.mesh = Some(handle); } @@ -137,6 +137,21 @@ impl ModelLoader { node } + + fn extensions() -> &'static [&'static str] { + &[ + "gltf", "glb" + ] + } + + fn does_support_file(path: &str) -> bool { + match Path::new(path).extension().and_then(OsStr::to_str) { + Some(ext) => { + Self::extensions().contains(&ext) + }, + _ => false, + } + } } impl ResourceLoader for ModelLoader { @@ -150,82 +165,92 @@ impl ResourceLoader for ModelLoader { &[] } - fn load(&self, resource_manager: &mut ResourceManager, path: &str) -> Result, crate::LoaderError> { - // check if the file is supported by this loader - if !self.does_support_file(path) { - return Err(LoaderError::UnsupportedExtension(path.to_string())); - } + fn load(&self, resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture { + // cant use &str across async + let path = path.to_string(); - let mut parent_path = PathBuf::from(path); - parent_path.pop(); - let parent_path = parent_path.display().to_string(); - - let gltf = gltf::Gltf::open(path)?; - - let mut use_bin = false; - let buffers: Vec> = gltf.buffers().flat_map(|b| match b.source() { - gltf::buffer::Source::Bin => { - use_bin = true; - gltf.blob.as_deref().map(|v| v.to_vec()) - .ok_or(ModelLoaderError::MissingBin(path.to_string())) - }, - gltf::buffer::Source::Uri(uri) => util::gltf_read_buffer_uri(&parent_path, uri) - .map_err(ModelLoaderError::UriDecodingError), - }).collect(); - - let mut gltf_out = super::Gltf::default(); - - let mut context = GltfLoadContext { - resource_manager, - gltf: &gltf, - gltf_path: path, - gltf_parent_path: &parent_path, - buffers: &buffers, - }; - - let start_inst = Instant::now(); - let materials: Vec> = gltf.materials() - .map(|mat| ResHandle::with_data("", Material::from_gltf(&mut context, mat))) - .collect(); - let mat_time = Instant::now() - start_inst; - debug!("Loaded {} materials in {}s", materials.len(), mat_time.as_secs_f32()); - - for (idx, scene) in gltf.scenes().enumerate() { - let start_inst = Instant::now(); - let nodes: Vec = scene.nodes() - .map(|node| ModelLoader::process_node(&mut context, &materials, node)) - .collect(); - let node_time = Instant::now() - start_inst; - - debug!("Loaded {} nodes in the scene in {}s", nodes.len(), node_time.as_secs_f32()); - - for mesh in nodes.iter().map(|n| &n.mesh) { - if let Some(mesh) = mesh { - gltf_out.meshes.push(mesh.clone()); - } + Box::pin(async move { + // check if the file is supported by this loader + if !Self::does_support_file(&path) { + return Err(LoaderError::UnsupportedExtension(path.to_string())); } - let scene = GltfScene { - nodes, - }; - let scene = ResHandle::with_data(&format!("{}:Scene{}", path, idx), scene); - gltf_out.scenes.push(scene); - } + let mut parent_path = PathBuf::from(&path); + parent_path.pop(); + let parent_path = parent_path.display().to_string(); - gltf_out.materials = materials; + let gltf = gltf::Gltf::open(&path)?; + + let mut use_bin = false; + let buffers: Vec> = gltf.buffers().flat_map(|b| match b.source() { + gltf::buffer::Source::Bin => { + use_bin = true; + gltf.blob.as_deref().map(|v| v.to_vec()) + .ok_or(ModelLoaderError::MissingBin(path.to_string())) + }, + gltf::buffer::Source::Uri(uri) => util::gltf_read_buffer_uri(&parent_path, uri) + .map_err(ModelLoaderError::UriDecodingError), + }).collect(); + + let mut gltf_out = super::Gltf::default(); + + let mut context = GltfLoadContext { + resource_manager: resource_manager.clone(), + gltf: &gltf, + gltf_path: &path, + gltf_parent_path: &parent_path, + buffers: &buffers, + }; + + let start_inst = Instant::now(); + let materials: Vec> = gltf.materials() + .map(|mat| ResHandle::new_ready(None, Material::from_gltf(&mut context, mat))) + .collect(); + let mat_time = Instant::now() - start_inst; + debug!("Loaded {} materials in {}s", materials.len(), mat_time.as_secs_f32()); + + for (_idx, scene) in gltf.scenes().enumerate() { + let start_inst = Instant::now(); + let nodes: Vec = scene.nodes() + .map(|node| ModelLoader::process_node(&mut context, &materials, node)) + .collect(); + let node_time = Instant::now() - start_inst; - Ok(Arc::new(ResHandle::with_data(path, gltf_out))) + debug!("Loaded {} nodes in the scene in {}s", nodes.len(), node_time.as_secs_f32()); + + for mesh in nodes.iter().map(|n| &n.mesh) { + if let Some(mesh) = mesh { + gltf_out.meshes.push(mesh.clone()); + } + } + + let scene = GltfScene { + nodes, + }; + let scene = ResHandle::new_ready(Some(path.as_str()), scene); + gltf_out.scenes.push(scene); + } + + gltf_out.materials = materials; + + Ok(Box::new(gltf_out) as Box) + }) } #[allow(unused_variables)] - fn load_bytes(&self, resource_manager: &mut ResourceManager, bytes: Vec, offset: usize, length: usize) -> Result, LoaderError> { + fn load_bytes(&self, resource_manager: ResourceManager, bytes: Vec, offset: usize, length: usize) -> PinedBoxLoaderFuture { todo!() } + + fn create_erased_handle(&self) -> Arc { + Arc::from(ResHandle::::new_loading(None)) + } } #[cfg(test)] mod tests { - use crate::{gltf::Gltf, ResourceLoader}; + use crate::tests::busy_wait_resource; + use super::*; fn test_file_path(path: &str) -> String { @@ -238,10 +263,9 @@ mod tests { fn test_loading() { let path = test_file_path("texture-embedded.gltf"); - let mut manager = ResourceManager::new(); - let loader = ModelLoader::default(); - let gltf = loader.load(&mut manager, &path).unwrap(); - let gltf = Arc::downcast::>(gltf.as_arc_any()).unwrap(); + let manager = ResourceManager::new(); + let gltf = manager.request::(&path).unwrap(); + busy_wait_resource(&gltf, 15.0); let gltf = gltf.data_ref().unwrap(); assert_eq!(gltf.scenes.len(), 1); diff --git a/lyra-resource/src/gltf/material.rs b/lyra-resource/src/gltf/material.rs index 1adeed0..a497cee 100644 --- a/lyra-resource/src/gltf/material.rs +++ b/lyra-resource/src/gltf/material.rs @@ -271,7 +271,7 @@ impl Material { wrap_w: WrappingMode::ClampToEdge, }; - let handler = ResHandle::with_data("", Texture { + let handler = ResHandle::new_ready(None, Texture { image: tex_img, sampler: Some(samp), }); diff --git a/lyra-resource/src/lib.rs b/lyra-resource/src/lib.rs index 196e75d..0f907a5 100644 --- a/lyra-resource/src/lib.rs +++ b/lyra-resource/src/lib.rs @@ -1,18 +1,17 @@ -pub mod resource_manager; +mod resource_manager; pub use resource_manager::*; -pub mod resource; +mod resource; pub use resource::*; -pub mod texture; +mod texture; pub use texture::*; pub mod loader; -pub use loader::*; pub mod gltf; -pub mod world_ext; +mod world_ext; pub use world_ext::*; pub(crate) mod util; diff --git a/lyra-resource/src/loader/image.rs b/lyra-resource/src/loader/image.rs index d9d0314..9ebb7c3 100644 --- a/lyra-resource/src/loader/image.rs +++ b/lyra-resource/src/loader/image.rs @@ -1,11 +1,12 @@ -use std::{fs::File, sync::Arc, io::Read}; +use std::{ffi::OsStr, path::Path, sync::Arc}; +use async_std::io::ReadExt; use image::ImageError; -use tracing::{debug, trace}; +use tracing::trace; -use crate::{resource::ResHandle, resource_manager::ResourceStorage, Image, ResourceManager}; +use crate::{Image, ResHandle, ResourceData, ResourceManager}; -use super::{LoaderError, ResourceLoader}; +use super::{LoaderError, PinedBoxLoaderFuture, ResourceLoader}; impl From for LoaderError { fn from(value: ImageError) -> Self { @@ -17,8 +18,8 @@ impl From for LoaderError { #[derive(Default)] pub struct ImageLoader; -impl ResourceLoader for ImageLoader { - fn extensions(&self) -> &[&str] { +impl ImageLoader { + fn extensions() -> &'static [&'static str] { &[ // the extensions of these are the names of the formats "bmp", "dds", "gif", "ico", "jpeg", "jpg", "png", "qoi", "tga", "tiff", "webp", @@ -31,6 +32,21 @@ impl ResourceLoader for ImageLoader { ] } + fn does_support_file(path: &str) -> bool { + match Path::new(path).extension().and_then(OsStr::to_str) { + Some(ext) => { + Self::extensions().contains(&ext) + }, + _ => false, + } + } +} + +impl ResourceLoader for ImageLoader { + fn extensions(&self) -> &[&str] { + Self::extensions() + } + fn mime_types(&self) -> &[&str] { &[ "image/bmp", "image/vnd.ms-dds", "image/gif", "image/x-icon", "image/jpeg", @@ -43,48 +59,55 @@ impl ResourceLoader for ImageLoader { ] } - fn load(&self, _resource_manager: &mut ResourceManager, path: &str) -> Result, LoaderError> { - // check if the file is supported by this loader - if !self.does_support_file(path) { - return Err(LoaderError::UnsupportedExtension(path.to_string())); - } + fn load(&self, _resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture { + let path = path.to_string(); + Box::pin(async move { + // check if the file is supported by this loader + if !Self::does_support_file(&path) { + return Err(LoaderError::UnsupportedExtension(path.to_string())); + } - // read file bytes - let mut file = File::open(path)?; - let mut buf = vec![]; - file.read_to_end(&mut buf)?; + // read file bytes + let mut file = async_std::fs::File::open(path).await?; + let mut buf = vec![]; + file.read_to_end(&mut buf).await?; - // load the image and construct Resource - let image = image::load_from_memory(&buf) - .map_err(|e| match e { - ImageError::IoError(e) => LoaderError::IoError(e), - _ => LoaderError::DecodingError(e.into()), - })?; - let image = Image::from(image); - let res = ResHandle::with_data(path, image); - - Ok(Arc::new(res)) + // load the image and construct Resource + let image = image::load_from_memory(&buf) + .map_err(|e| match e { + ImageError::IoError(e) => LoaderError::IoError(e), + _ => LoaderError::DecodingError(e.into()), + })?; + let image = Image::from(image); + let image = Box::new(image) as Box; + + Ok(image) + }) } - fn load_bytes(&self, _resource_manager: &mut ResourceManager, bytes: Vec, offset: usize, length: usize) -> Result, LoaderError> { - trace!("Loading {} bytes as an image", length); - - let image = image::load_from_memory(&bytes[offset..(length-offset)]) - .map_err(|e| match e { - ImageError::IoError(e) => LoaderError::IoError(e), - _ => LoaderError::DecodingError(e.into()), - })?; - let image = Image::from(image); - let res = ResHandle::with_data(&uuid::Uuid::new_v4().to_string(), image); - - debug!("Finished loading image of {} bytes", length); + fn load_bytes(&self, _resource_manager: ResourceManager, bytes: Vec, offset: usize, length: usize) -> PinedBoxLoaderFuture { + Box::pin(async move { + trace!("Loading {} bytes as an image", length); + + let image = image::load_from_memory(&bytes[offset..(length-offset)]) + .map_err(|e| match e { + ImageError::IoError(e) => LoaderError::IoError(e), + _ => LoaderError::DecodingError(e.into()), + })?; + let image = Image::from(image); + Ok(Box::new(image) as Box) + }) + } - Ok(Arc::new(res)) + fn create_erased_handle(&self) -> Arc { + Arc::from(ResHandle::::new_loading(None)) } } #[cfg(test)] mod tests { + use async_std::task; + use super::*; fn get_image(path: &str) -> String { @@ -102,15 +125,24 @@ mod tests { /// Tests loading an image #[test] fn image_load() { - let mut manager = ResourceManager::new(); + let manager = ResourceManager::new(); let loader = ImageLoader::default(); - loader.load(&mut manager, &get_image("squiggles.png")).unwrap(); + + task::block_on(async move { + let r = loader.load(manager, &get_image("squiggles.png")).await.unwrap(); + let a = r.as_ref(); + a.as_any().downcast_ref::().unwrap(); + }); } #[test] fn image_load_unsupported() { - let mut manager = ResourceManager::new(); + let manager = ResourceManager::new(); let loader = ImageLoader::default(); - assert!(loader.load(&mut manager, &get_image("squiggles.gltf")).is_err()); + + task::block_on(async move { + // this file doesn't exist and is also not supported + assert!(loader.load(manager, &get_image("squiggles.jfeh")).await.is_err()) + }); } } \ No newline at end of file diff --git a/lyra-resource/src/loader/mod.rs b/lyra-resource/src/loader/mod.rs index 5e2f0f6..38aa23e 100644 --- a/lyra-resource/src/loader/mod.rs +++ b/lyra-resource/src/loader/mod.rs @@ -1,10 +1,11 @@ -pub mod image; +mod image; +pub use image::*; -use std::{io, sync::Arc, path::Path, ffi::OsStr}; +use std::{ffi::OsStr, future::Future, io, path::Path, pin::Pin, sync::Arc}; use thiserror::Error; -use crate::{resource_manager::ResourceStorage, ResourceManager}; +use crate::{resource_manager::ResourceStorage, ResourceData, ResourceManager}; #[derive(Error, Debug)] pub enum LoaderError { @@ -28,7 +29,9 @@ impl From for LoaderError { } } -pub trait ResourceLoader { +pub type PinedBoxLoaderFuture = Pin, LoaderError>> + Send>>; + +pub trait ResourceLoader: Send + Sync { /// Returns the extensions that this loader supports. Does not include the `.` fn extensions(&self) -> &[&str]; /// Returns the mime types that this loader supports. @@ -50,9 +53,21 @@ pub trait ResourceLoader { } /// Load a resource from a path. - fn load(&self, resource_manager: &mut ResourceManager, path: &str) -> Result, LoaderError>; + fn load(&self, resource_manager: ResourceManager, path: &str) -> PinedBoxLoaderFuture; /// Load a resource from bytes. - fn load_bytes(&self, resource_manager: &mut ResourceManager, bytes: Vec, offset: usize, length: usize) -> Result, LoaderError>; + fn load_bytes(&self, resource_manager: ResourceManager, bytes: Vec, offset: usize, length: usize) -> PinedBoxLoaderFuture; + + /// Creates and returns a `ResHandle` in a loading state that stores the type that this + /// loader returns. + /// + /// This is very simple to implement, you can use this as a template: + /// ```nobuild + /// fn create_erased_handle(&self) -> Arc { + /// // Change the type of the reshandle to match the loader. + /// Arc::from(ResHandle::::new_loading(None)) + /// } + /// ``` + fn create_erased_handle(&self) -> Arc; } diff --git a/lyra-resource/src/resource.rs b/lyra-resource/src/resource.rs index 3b7610b..926daeb 100644 --- a/lyra-resource/src/resource.rs +++ b/lyra-resource/src/resource.rs @@ -1,37 +1,64 @@ -use std::{any::Any, sync::{Arc, RwLock}}; +use std::{any::{Any, TypeId}, marker::PhantomData, sync::{Arc, RwLock}}; use lyra_ecs::Component; -use crate::lyra_engine; +use crate::{loader::LoaderError, lyra_engine}; use uuid::Uuid; use crate::ResourceStorage; -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +/// A trait that that each resource type should implement. +pub trait ResourceData: Send + Sync + Any + 'static { + fn as_any(&self) -> &dyn Any; + fn as_any_mut(&mut self) -> &mut dyn Any; + fn type_id(&self) -> TypeId; +} + +impl ResourceData for T { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn type_id(&self) -> TypeId { + TypeId::of::() + } +} + pub enum ResourceState { Loading, - Ready, + Error(Arc), + Ready(Box), } pub struct ResourceDataRef<'a, T> { guard: std::sync::RwLockReadGuard<'a, Resource>, } -impl<'a, T> std::ops::Deref for ResourceDataRef<'a, T> { +impl<'a, T: 'static> std::ops::Deref for ResourceDataRef<'a, T> { type Target = T; fn deref(&self) -> &Self::Target { - // safety: this struct must only be created if the resource is loaded - self.guard.data.as_ref().unwrap() + match &self.guard.state { + ResourceState::Ready(d) => { + // for some reason, if I didn't use `.as_ref`, the downcast would fail. + let d = d.as_ref().as_any(); + d.downcast_ref::().unwrap() + }, + _ => unreachable!() // ResHandler::data_ref shouldn't allow this to run + } } } pub struct Resource { - path: String, - pub(crate) data: Option, pub(crate) version: usize, pub(crate) state: ResourceState, uuid: Uuid, + path: Option, pub(crate) is_watched: bool, + _marker: PhantomData, } /// A handle to a resource. @@ -51,16 +78,31 @@ impl Clone for ResHandle { } } -impl ResHandle { - /// Create the resource with data, its assumed the state is `Ready` - pub fn with_data(path: &str, data: T) -> Self { +impl ResHandle { + pub fn new_loading(path: Option<&str>) -> Self { let res_version = Resource { - path: path.to_string(), - data: Some(data), version: 0, - state: ResourceState::Ready, + path: path.map(str::to_string), + state: ResourceState::Loading, uuid: Uuid::new_v4(), is_watched: false, + _marker: PhantomData:: + }; + + Self { + data: Arc::new(RwLock::new(res_version)), + } + } + + /// Create the resource with data, its assumed the state is `Ready` + pub fn new_ready(path: Option<&str>, data: T) -> Self { + let res_version = Resource { + version: 0, + path: path.map(str::to_string), + state: ResourceState::Ready(Box::new(data)), + uuid: Uuid::new_v4(), + is_watched: false, + _marker: PhantomData:: }; Self { @@ -77,20 +119,14 @@ impl ResHandle { /// Returns a boolean indicating if this resource is loaded pub fn is_loaded(&self) -> bool { let d = self.data.read().expect("Resource mutex was poisoned!"); - d.state == ResourceState::Ready + matches!(d.state, ResourceState::Ready(_)) } /// Returns the current state of the resource. - pub fn state(&self) -> ResourceState { + /* pub fn state(&self) -> &ResourceState { let d = self.data.read().expect("Resource mutex was poisoned!"); - d.state - } - - /// Returns the path that the resource was loaded from. - pub fn path(&self) -> String { - let d = self.data.read().expect("Resource mutex was poisoned!"); - d.path.to_string() - } + &d.state + } */ /// Returns the uuid of the resource. pub fn uuid(&self) -> Uuid { @@ -98,6 +134,11 @@ impl ResHandle { d.uuid } + pub fn path(&self) -> Option { + let d = self.data.read().expect("Resource mutex was poisoned!"); + d.path.clone() + } + /// Retrieves the current version of the resource. This gets incremented when the resource /// is reloaded. pub fn version(&self) -> usize { @@ -140,22 +181,18 @@ impl ResourceStorage for ResHandle { w.is_watched = watched; } - fn path(&self) -> String { - self.path() - } - fn version(&self) -> usize { self.version() } - fn state(&self) -> ResourceState { - self.state() - } - fn uuid(&self) -> Uuid { self.uuid() } + fn path(&self) -> Option { + self.path() + } + fn is_watched(&self) -> bool { self.is_watched() } @@ -163,4 +200,11 @@ impl ResourceStorage for ResHandle { fn is_loaded(&self) -> bool { self.is_loaded() } + + fn set_state(&self, new: ResourceState) { + let mut d = self.data.write().expect("Resource mutex was poisoned!"); + d.state = new; + } + + } \ No newline at end of file diff --git a/lyra-resource/src/resource_manager.rs b/lyra-resource/src/resource_manager.rs index 2069b5e..f184148 100644 --- a/lyra-resource/src/resource_manager.rs +++ b/lyra-resource/src/resource_manager.rs @@ -1,12 +1,13 @@ -use std::{sync::{Arc, RwLock}, collections::HashMap, any::Any, path::Path, time::Duration}; +use std::{any::Any, collections::HashMap, path::Path, sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, time::Duration}; +use async_std::task; use crossbeam::channel::Receiver; use notify::{Watcher, RecommendedWatcher}; use notify_debouncer_full::{DebouncedEvent, FileIdMap}; use thiserror::Error; use uuid::Uuid; -use crate::{gltf::ModelLoader, loader::{image::ImageLoader, LoaderError, ResourceLoader}, resource::ResHandle, ResourceState}; +use crate::{gltf::ModelLoader, loader::{ImageLoader, LoaderError, ResourceLoader}, resource::ResHandle, ResourceState}; /// A trait for type erased storage of a resource. /// Implemented for [`ResHandle`] @@ -19,24 +20,30 @@ pub trait ResourceStorage: Send + Sync + Any + 'static { /// This is used internally. fn set_watched(&self, watched: bool); - fn path(&self) -> String; fn version(&self) -> usize; - fn state(&self) -> ResourceState; fn uuid(&self) -> Uuid; + fn path(&self) -> Option; fn is_watched(&self) -> bool; fn is_loaded(&self) -> bool; + fn set_state(&self, new: ResourceState); } #[derive(Error, Debug)] pub enum RequestError { #[error("{0}")] Loader(LoaderError), + #[error("The file extension is unsupported: '{0}'")] UnsupportedFileExtension(String), + #[error("The mimetype is unsupported: '{0}'")] UnsupportedMime(String), + #[error("The identifier is not found: '{0}'")] IdentNotFound(String), + + #[error("The resource was not loaded from a path so cannot be reloaded")] + NoReloadPath } impl From for RequestError { @@ -45,51 +52,79 @@ impl From for RequestError { } } -/// A struct that stores all Manager data. This is requried for sending -//struct ManagerStorage - -/// A struct that +/// A struct that stores some things used for watching resources. pub struct ResourceWatcher { debouncer: Arc>>, events_recv: Receiver, Vec>>, } -pub struct ResourceManager { +/// The state of the ResourceManager +pub struct ResourceManagerState { resources: HashMap>, - uuid_resources: HashMap>, loaders: Vec>, watchers: HashMap, } +/// The ResourceManager +/// +/// This exists since we need the manager to be `Send + Sync`. +#[derive(Clone)] +pub struct ResourceManager { + inner: Arc>, +} + impl Default for ResourceManager { fn default() -> Self { - Self::new() + Self { + inner: Arc::new(RwLock::new( + ResourceManagerState { + resources: HashMap::new(), + loaders: vec![ Arc::new(ImageLoader), Arc::new(ModelLoader) ], + watchers: HashMap::new(), + } + )) + } } } impl ResourceManager { pub fn new() -> Self { - Self { - resources: HashMap::new(), - uuid_resources: HashMap::new(), - loaders: vec![ Arc::new(ImageLoader), Arc::new(ModelLoader) ], - watchers: HashMap::new(), - } + Self::default() + } + + /// Retrieves a non-mutable guard of the manager's state. + pub fn state(&self) -> RwLockReadGuard { + self.inner.read().unwrap() + } + + /// Retrieves a mutable guard of the manager's state. + pub fn state_mut(&self) -> RwLockWriteGuard { + self.inner.write().unwrap() } /// Registers a loader to the manager. - pub fn register_loader(&mut self) + pub fn register_loader(&self) where L: ResourceLoader + Default + 'static { - self.loaders.push(Arc::new(L::default())); + let mut state = self.state_mut(); + state.loaders.push(Arc::new(L::default())); } - pub fn request(&mut self, path: &str) -> Result, RequestError> + /// Request a resource at `path`. + /// + /// When a resource for a path is requested for the first time, the resource will be loaded + /// and cached. In the future, the cached version will be returned. There is only ever one copy + /// of the resource's data in memory at a time. + /// + /// Loading resources is done asynchronously on a task spawned by `async-std`. You can use the + /// handle to check if the resource is loaded. + pub fn request(&self, path: &str) -> Result, RequestError> where T: Send + Sync + Any + 'static { - match self.resources.get(&path.to_string()) { + let mut state = self.state_mut(); + match state.resources.get(&path.to_string()) { Some(res) => { let res = res.clone().as_arc_any(); let res: Arc> = res.downcast::>().expect("Failure to downcast resource"); @@ -98,22 +133,33 @@ impl ResourceManager { Ok(res) }, None => { - if let Some(loader) = self.loaders.iter() + if let Some(loader) = state.loaders.iter() .find(|l| l.does_support_file(path)) { // Load the resource and store it let loader = Arc::clone(loader); // stop borrowing from self - let res = loader.load(self, path)?; - let res: Arc = Arc::from(res); - self.resources.insert(path.to_string(), res.clone()); + let res = loader.load(self.clone(), path); + + let handle = ResHandle::::new_loading(Some(path)); + + let thand = handle.clone(); + task::spawn(async move { + match res.await { + Ok(data) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Ready(data); + } + Err(err) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Error(Arc::new(err)); + } + } + }); - // cast Arc to Arc - let res = res.as_arc_any(); - let res = res.downcast::>() - .expect("Failure to downcast resource! Does the loader return an `Arc>`?"); - let res = ResHandle::::clone(&res); + let res: Arc = Arc::from(handle.clone()); + state.resources.insert(path.to_string(), res); - Ok(res) + Ok(handle) } else { Err(RequestError::UnsupportedFileExtension(path.to_string())) } @@ -121,28 +167,42 @@ impl ResourceManager { } } - /// Request a resource without downcasting to a ResHandle. + /// Request a resource without downcasting to a `ResHandle`. /// Whenever you're ready to downcast, you can do so like this: /// ```compile_fail /// let arc_any = res_arc.as_arc_any(); /// let res: Arc> = res.downcast::>().expect("Failure to downcast resource"); /// ``` - pub fn request_raw(&mut self, path: &str) -> Result, RequestError> { - match self.resources.get(&path.to_string()) { + pub fn request_raw(&self, path: &str) -> Result, RequestError> { + let inner = self.inner.write().unwrap(); + match inner.resources.get(&path.to_string()) { Some(res) => { Ok(res.clone()) }, None => { - if let Some(loader) = self.loaders.iter() + if let Some(loader) = inner.loaders.iter() .find(|l| l.does_support_file(path)) { // Load the resource and store it let loader = Arc::clone(loader); // stop borrowing from self - let res = loader.load(self, path)?; - let res: Arc = Arc::from(res); - self.resources.insert(path.to_string(), res.clone()); + let res = loader.load(self.clone(), path); - Ok(res) + let handle = loader.create_erased_handle(); + //let handle = ResHandle::::new_loading(); + + let thand = handle.clone(); + task::spawn(async move { + match res.await { + Ok(data) => { + thand.set_state(ResourceState::Ready(data)); + } + Err(err) => { + thand.set_state(ResourceState::Error(Arc::new(err))); + } + } + }); + + Ok(handle) } else { Err(RequestError::UnsupportedFileExtension(path.to_string())) } @@ -154,16 +214,18 @@ impl ResourceManager { /// /// The resource cannot be requested with [`ResourceManager::request`], it can only be /// retrieved with [`ResourceManager::request_uuid`]. - pub fn store_uuid(&mut self, res: ResHandle) { - self.uuid_resources.insert(res.uuid(), Arc::new(res)); + pub fn store_uuid(&self, res: ResHandle) { + let mut state = self.state_mut(); + state.resources.insert(res.uuid().to_string(), Arc::new(res)); } /// Request a resource via its uuid. /// /// Returns `None` if the resource was not found. The resource must of had been /// stored with [`ResourceManager::request`] to return `Some`. - pub fn request_uuid(&mut self, uuid: &Uuid) -> Option> { - match self.uuid_resources.get(uuid) { + pub fn request_uuid(&self, uuid: &Uuid) -> Option> { + let state = self.state(); + match state.resources.get(&uuid.to_string()) { Some(res) => { let res = res.clone().as_arc_any(); let res: Arc> = res.downcast::>().expect("Failure to downcast resource"); @@ -182,36 +244,47 @@ impl ResourceManager { /// * `bytes` - The bytes to store. /// /// Returns: The `Arc` to the now stored resource - pub fn load_bytes(&mut self, ident: &str, mime_type: &str, bytes: Vec, offset: usize, length: usize) -> Result, RequestError> + pub fn load_bytes(&self, ident: &str, mime_type: &str, bytes: Vec, offset: usize, length: usize) -> Result, RequestError> where T: Send + Sync + Any + 'static { - if let Some(loader) = self.loaders.iter() + let mut state = self.state_mut(); + if let Some(loader) = state.loaders.iter() .find(|l| l.does_support_mime(mime_type)) { let loader = loader.clone(); - let res = loader.load_bytes(self, bytes, offset, length)?; - let res: Arc = Arc::from(res); - self.resources.insert(ident.to_string(), res.clone()); - // code here... + let res = loader.load_bytes(self.clone(), bytes, offset, length); - // cast Arc to Arc - let res = res.as_arc_any(); - let res = res.downcast::>() - .expect("Failure to downcast resource! Does the loader return an `Arc>`?"); - let res = ResHandle::::clone(&res); + let handle = ResHandle::::new_loading(None); + let thand = handle.clone(); + task::spawn(async move { + match res.await { + Ok(data) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Ready(data); + } + Err(err) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Error(Arc::new(err)); + } + } + }); - Ok(res) + let res: Arc = Arc::from(handle.clone()); + state.resources.insert(ident.to_string(), res); + + Ok(handle) } else { Err(RequestError::UnsupportedMime(mime_type.to_string())) } } /// Requests bytes from the manager. - pub fn request_loaded_bytes(&mut self, ident: &str) -> Result>, RequestError> + pub fn request_loaded_bytes(&self, ident: &str) -> Result>, RequestError> where T: Send + Sync + Any + 'static { - match self.resources.get(&ident.to_string()) { + let state = self.state(); + match state.resources.get(&ident.to_string()) { Some(res) => { let res = res.clone().as_arc_any(); let res = res.downcast::>().expect("Failure to downcast resource"); @@ -225,7 +298,7 @@ impl ResourceManager { } /// Start watching a path for changes. Returns a mspc channel that will send events - pub fn watch(&mut self, path: &str, recursive: bool) -> notify::Result, Vec>>> { + pub fn watch(&self, path: &str, recursive: bool) -> notify::Result, Vec>>> { let (send, recv) = crossbeam::channel::bounded(15); let mut watcher = notify_debouncer_full::new_debouncer(Duration::from_millis(1000), None, send)?; @@ -241,9 +314,10 @@ impl ResourceManager { events_recv: recv.clone(), }; - self.watchers.insert(path.to_string(), watcher); + let mut state = self.state_mut(); + state.watchers.insert(path.to_string(), watcher); - let res = self.resources.get(&path.to_string()) + let res = state.resources.get(&path.to_string()) .expect("The path that was watched has not been loaded as a resource yet"); res.set_watched(true); @@ -251,13 +325,14 @@ impl ResourceManager { } /// Stops watching a path - pub fn stop_watching(&mut self, path: &str) -> notify::Result<()> { - if let Some(watcher) = self.watchers.get(path) { + pub fn stop_watching(&self, path: &str) -> notify::Result<()> { + let state = self.state(); + if let Some(watcher) = state.watchers.get(path) { let mut watcher = watcher.debouncer.write().unwrap(); watcher.watcher().unwatch(Path::new(path))?; // unwrap is safe since only loaded resources can be watched - let res = self.resources.get(&path.to_string()).unwrap(); + let res = state.resources.get(&path.to_string()).unwrap(); res.set_watched(false); } @@ -267,43 +342,50 @@ impl ResourceManager { /// Returns a mspc receiver for watcher events of a specific path. The path must already /// be watched with [`ResourceManager::watch`] for this to return `Some`. pub fn watcher_event_recv(&self, path: &str) -> Option, Vec>>> { - self.watchers.get(&path.to_string()) + let state = self.state(); + state.watchers.get(&path.to_string()) .map(|w| w.events_recv.clone()) } - /// Reloads a resource. The data inside the resource will be updated, the state may - pub fn reload(&mut self, resource: ResHandle) -> Result<(), RequestError> + /// Trigger a reload of a resource. + /// + /// The version of the resource will be incremented by one. + /// + /// > Note: Since reloading is done asynchronously, the reloaded data will not be immediately + /// accessible. Until the resource is reloaded, the previous data will stay inside of + /// the handle. + pub fn reload(&self, resource: ResHandle) -> Result<(), RequestError> where T: Send + Sync + Any + 'static { - let path = resource.path(); - if let Some(loader) = self.loaders.iter() - .find(|l| l.does_support_file(&path)) { - println!("got loader"); - let loader = Arc::clone(loader); // stop borrowing from self - let loaded = loader.load(self, &path)?; - let loaded = loaded.as_arc_any(); - - let loaded = loaded.downcast::>() - .unwrap(); - let loaded = match Arc::try_unwrap(loaded) { - Ok(v) => v, - Err(_) => panic!("Seems impossible that this would happen, the resource was just loaded!"), - }; - let loaded = loaded.data; - let loaded = match Arc::try_unwrap(loaded) { - Ok(v) => v, - Err(_) => panic!("Seems impossible that this would happen, the resource was just loaded!"), - }; - let loaded = loaded.into_inner().unwrap(); + let state = self.state(); - let res_lock = &resource.data; + let path = resource.path() + .ok_or(RequestError::NoReloadPath)?; + if let Some(loader) = state.loaders.iter() + .find(|l| l.does_support_file(&path)) { + let loader = Arc::clone(loader); // stop borrowing from self + let res = loader.load(self.clone(), &path); + + /* let res_lock = &resource.data; let mut res_lock = res_lock.write().unwrap(); - let version = res_lock.version; + res_lock.state = ResourceState::Loading; + drop(res_lock); */ - res_lock.data = loaded.data; - res_lock.state = loaded.state; - res_lock.version = version + 1; + let thand = resource.clone(); + task::spawn(async move { + match res.await { + Ok(data) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Ready(data); + d.version += 1; + } + Err(err) => { + let mut d = thand.data.write().unwrap(); + d.state = ResourceState::Error(Arc::new(err)); + } + } + }); } Ok(()) @@ -311,10 +393,12 @@ impl ResourceManager { } #[cfg(test)] -mod tests { - use std::io; +pub(crate) mod tests { + use std::{io, ops::Deref}; - use crate::{Texture, ResourceState}; + use instant::Instant; + + use crate::{Image, ResourceData, Texture}; use super::*; @@ -324,37 +408,81 @@ mod tests { format!("{manifest}/test_files/img/{path}") } + pub(crate) fn busy_wait_resource(handle: &ResHandle, timeout: f32) { + let start = Instant::now(); + while !handle.is_loaded() { + // loop until the image is loaded + let now = Instant::now(); + + let diff = now - start; + + if diff.as_secs_f32() >= timeout { + panic!("Image never loaded"); + } + } + } + + pub(crate) fn busy_wait_resource_reload(handle: &ResHandle, timeout: f32) { + let version = handle.version(); + let start = Instant::now(); + + while !handle.is_loaded() || handle.version() == version { + // loop until the image is loaded + let now = Instant::now(); + + let diff = now - start; + + if diff.as_secs_f32() >= timeout { + panic!("Image never loaded"); + } + } + } + #[test] fn load_image() { - let mut man = ResourceManager::new(); - let res = man.request::(&get_image("squiggles.png")).unwrap(); - assert_eq!(res.state(), ResourceState::Ready); - let img = res.data_ref(); - img.unwrap(); + let man = ResourceManager::new(); + let res = man.request::(&get_image("squiggles.png")).unwrap(); + assert!(!res.is_loaded()); + + busy_wait_resource(&res, 10.0); + + // shouldn't panic because of the loop + res.data_ref().unwrap(); } /// Ensures that only one copy of the same data was made #[test] fn ensure_single() { - let mut man = ResourceManager::new(); + let man = ResourceManager::new(); let res = man.request::(&get_image("squiggles.png")).unwrap(); - assert_eq!(Arc::strong_count(&res.data), 2); + assert_eq!(Arc::strong_count(&res.data), 3); let resagain = man.request::(&get_image("squiggles.png")).unwrap(); - assert_eq!(Arc::strong_count(&resagain.data), 3); + assert_eq!(Arc::strong_count(&resagain.data), 4); } /// Ensures that an error is returned when a file that doesn't exist is requested #[test] fn ensure_none() { - let mut man = ResourceManager::new(); - let res = man.request::(&get_image("squigglesfff.png")); - let err = res.err().unwrap(); + let man = ResourceManager::new(); + let res = man.request::(&get_image("squigglesfff.png")).unwrap(); + //let err = res.err().unwrap(); + + // 1 second should be enough to run into an error + std::thread::sleep(Duration::from_secs(1)); + //busy_wait_resource(&res, 10.0); + let state = &res.data.read().unwrap().state; assert!( - match err { + match state { // make sure the error is NotFound - RequestError::Loader(LoaderError::IoError(e)) if e.kind() == io::ErrorKind::NotFound => true, + //RequestError::Loader(LoaderError::IoError(e)) if e.kind() == io::ErrorKind::NotFound => true, + ResourceState::Error(err) => { + match err.deref() { + LoaderError::IoError(e) if e.kind() == io::ErrorKind::NotFound => true, + _ => false, + } + }, _ => false } ); @@ -362,17 +490,18 @@ mod tests { #[test] fn reload_image() { - let mut man = ResourceManager::new(); + let man = ResourceManager::new(); let res = man.request::(&get_image("squiggles.png")).unwrap(); - assert_eq!(res.state(), ResourceState::Ready); + busy_wait_resource(&res, 10.0); let img = res.data_ref(); img.unwrap(); - println!("Path = {}", res.path()); man.reload(res.clone()).unwrap(); + busy_wait_resource_reload(&res, 10.0); assert_eq!(res.version(), 1); man.reload(res.clone()).unwrap(); + busy_wait_resource_reload(&res, 10.0); assert_eq!(res.version(), 2); } @@ -382,9 +511,9 @@ mod tests { let image_path = get_image("squiggles_test.png"); std::fs::copy(orig_path, &image_path).unwrap(); - let mut man = ResourceManager::new(); + let man = ResourceManager::new(); let res = man.request::(&image_path).unwrap(); - assert_eq!(res.state(), ResourceState::Ready); + busy_wait_resource(&res, 10.0); let img = res.data_ref(); img.unwrap(); diff --git a/lyra-resource/src/world_ext.rs b/lyra-resource/src/world_ext.rs index 2db2b04..39856ca 100644 --- a/lyra-resource/src/world_ext.rs +++ b/lyra-resource/src/world_ext.rs @@ -4,7 +4,7 @@ use crossbeam::channel::Receiver; use lyra_ecs::World; use notify_debouncer_full::DebouncedEvent; -use crate::{RequestError, ResHandle, ResourceLoader, ResourceManager}; +use crate::{loader::ResourceLoader, RequestError, ResHandle, ResourceManager}; pub trait WorldAssetExt { /// Register a resource loader with the resource manager. @@ -38,7 +38,7 @@ impl WorldAssetExt for World { where L: ResourceLoader + Default + 'static { - let mut man = self.get_resource_or_default::(); + let man = self.get_resource_or_default::(); man.register_loader::(); } @@ -46,17 +46,17 @@ impl WorldAssetExt for World { where T: Send + Sync + Any + 'static { - let mut man = self.get_resource_or_default::(); + let man = self.get_resource_or_default::(); man.request(path) } fn watch_res(&mut self, path: &str, recursive: bool) -> notify::Result, Vec>>> { - let mut man = self.get_resource_or_default::(); + let man = self.get_resource_or_default::(); man.watch(path, recursive) } fn stop_watching_res(&mut self, path: &str) -> notify::Result<()> { - let mut man = self.get_resource_or_default::(); + let man = self.get_resource_or_default::(); man.stop_watching(path) } @@ -69,7 +69,7 @@ impl WorldAssetExt for World { where T: Send + Sync + Any + 'static { - let mut man = self.get_resource_or_default::(); + let man = self.get_resource_or_default::(); man.reload(resource) } } \ No newline at end of file diff --git a/lyra-scripting/src/lua/loader.rs b/lyra-scripting/src/lua/loader.rs index eea2efd..9571c3c 100644 --- a/lyra-scripting/src/lua/loader.rs +++ b/lyra-scripting/src/lua/loader.rs @@ -23,7 +23,7 @@ impl ResourceLoader for LuaLoader { fn load(&self, _resource_manager: &mut lyra_resource::ResourceManager, path: &str) -> Result, lyra_resource::LoaderError> { let bytes = std::fs::read(path)?; - let s = ResHandle::with_data(path, LuaScript { + let s = ResHandle::new_ready(path, LuaScript { bytes }); @@ -34,7 +34,7 @@ impl ResourceLoader for LuaLoader { let end = offset + length; let bytes = bytes[offset..end].to_vec(); - let s = ResHandle::with_data("from bytes", LuaScript { + let s = ResHandle::new_ready("from bytes", LuaScript { bytes });