SeanOMik
/
wgsl-preprocessor
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Implement imports of structs and type aliases #2

Merged
SeanOMik merged 3 commits from feat/struct-imports into main 2024-08-30 23:57:14 +00:00
Conversation 0 Commits 3 Files Changed 4 +186 -62
4 changed files with 186 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a9b0eb6abf - Show all commits

42
.vscode/launch.json vendored
View File

@ -7,15 +7,15 @@
{
"type": "lldb",
"request": "launch",
"name": "Debug shader_prepoc",
"name": "Debug wgsl_preprocessor",
"cargo": {
"args": [
"build",
//"--manifest-path", "${workspaceFolder}/examples/testbed/Cargo.toml"
"--bin=shader_prepoc",
"--bin=wgsl_preprocessor",
],
"filter": {
"name": "shader_prepoc",
"name": "wgsl_preprocessor",
"kind": "bin"
}
},
@ -25,33 +25,55 @@
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in executable 'shader_prepoc'",
"name": "Debug unit tests in executable 'wgsl_preprocessor'",
"cargo": {
"args": [
"test",
"--no-run",
"--bin=shader_prepoc",
"--package=shader_prepoc"
"--bin=wgsl_preprocessor",
"--package=wgsl_preprocessor"
],
"filter": {
"name": "shader_prepoc",
"name": "wgsl_preprocessor",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug specific test in executable 'wgsl_preprocessor'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=wgsl_preprocessor",
"tests::import_struct_with_imported_fields",
"--",
"--exact --nocapture"
],
"filter": {
"name": "wgsl_preprocessor",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
/* {
"type": "lldb",
"request": "launch",
"name": "Test shader_prepoc",
"name": "Test wgsl_preprocessor",
"cargo": {
"args": [
"test",
"--bin=shader_prepoc",
"--bin=wgsl_preprocessor",
],
"filter": {
"name": "shader_prepoc",
"name": "wgsl_preprocessor",
"kind": "bin"
}
},

50
src/lib.rs
View File

@ -19,7 +19,7 @@ pub enum ExternalUsageType {
Function,
}
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct Import {
module: String,
imports: Vec<String>,
@ -50,6 +50,7 @@ pub struct Module {
src: String,
/// Constants that this module defines
pub vars: HashMap<String, Definition>,
pub structs: HashMap<String, Definition>,
/// Functions that this module defines
pub functions: HashMap<String, Definition>,
/// Imports of things per module
@ -326,36 +327,38 @@ fn main() -> vec4<f32> {
let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
.expect("failed to find main module def");
let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
let lights_mod = p.modules.get("engine::lights").unwrap();
assert!(lights_mod.structs.contains_key("Light"), "`Light` struct definition was not parsed");
let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
assert!(out.contains("struct Light"), "missing `Light` struct definition: {}", out);
}
/* #[test]
fn import_type_aliases() {
#[test]
fn import_struct_with_imported_fields() {
const TYPE_MODULE: &'static str =
r#"#define_module engine::types
struct Something {
val: f32
}"#;
const BINDINGS_MODULE: &'static str =
r#"#define_module engine::aliases
r#"#define_module engine::lights
#import engine::types::{Something}
struct Light {
some: Something,
intensity: f32
}
struct Lights {
num: u32,
data: array<Light>
}
alias LightArray = Lights;"#;
}"#;
const IMPORTING_MODULE: &'static str =
r#"#define_module engine::main
#import engine::aliases::{LightArray}
@group(0) @binding(0) var<uniform> lights: LightArray;
#import engine::lights::{Light}
fn main() -> vec4<f32> {
// imports must be used to be generated
let light_num = lights.num;
let base_light = Light(1.0);
return vec4<f32>(1.0);
}"#;
@ -367,15 +370,18 @@ fn main() -> vec4<f32> {
.init();
let mut p = Processor::new();
p.parse_module(TYPE_MODULE).unwrap()
.expect("failed to find bindings module def");
p.parse_module(BINDINGS_MODULE).unwrap()
.expect("failed to find bindings module def");
let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
.expect("failed to find main module def");
let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
let lights_mod = p.modules.get("engine::lights").unwrap();
assert!(lights_mod.structs.contains_key("Light"), "`Light` struct definition was not parsed");
assert!(out.contains("alias LightArray = Lights"), "missing use_anyway binding: {}", out);
let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
assert!(out.contains("struct Light"), "missing `Light` struct definition: {}", out);
assert!(out.contains("struct Lights"), "missing `Lights` struct definition: {}", out);
} */
}
assert!(out.contains("struct Something"), "missing `Something` struct definition: {}", out);
}
}

154
src/preprocessor.rs
View File

@ -63,6 +63,46 @@ pub enum PreprocessorError {
ConflictingImport { from_module: String, name: String },
}
fn get_external_var_requirement(module: &Module, pair: Pair<Rule>) -> Option<DefRequirement> {
let var_name = pair.as_str();
if pair.as_rule() == Rule::shader_external_variable {
let (module_name, name) = var_name.rsplit_once("::").unwrap();
if RESERVED_WORDS.contains(&name) {
return None;
}
// Find the module that this variable is from.
// Starts by checking if the full module path was used.
// If it was not, then find the full path of the module.
let module_full_name = if module.module_imports.contains(module_name) {
Some(module_name.to_string())
} else {
module.module_imports.iter()
.find(|m| {
m.ends_with(module_name)
}).cloned()
};
debug!("Binding module is {:?}", module_full_name);
Some(DefRequirement {
module: module_full_name,
name: name.to_string(),
})
} else { // The only possibility is `Rule::shader_type`
if RESERVED_WORDS.contains(&var_name) {
return None;
}
Some(DefRequirement {
module: None,
name: var_name.to_string(),
})
}
}
#[derive(Default)]
pub struct Processor {
pub modules: HashMap<String, Module>,
@ -73,28 +113,6 @@ impl Processor {
Self::default()
}
//
/* fn add_func_requirements(&self, found_requirements: &mut HashSet<String>, requirements: &mut Vec<DefRequirement>, fn_name: &str) {
if found_requirements.contains(fn_name) {
return;
}
found_requirements.insert(fn_name.to_string());
debug!("Found call to `{}`", fn_name);
// ignore reserved words
if RESERVED_WORDS.contains(&fn_name) {
return;
}
let req = DefRequirement {
// module is discovered later
module: None,
name: fn_name.to_string(),
};
requirements.push(req);
} */
#[instrument(fields(module = module.name), skip_all)]
fn get_imports_in_block(&mut self, module: &mut Module, block: Pair<Rule>, found_requirements: &mut HashSet<String>) -> Vec<DefRequirement> {
let mut requirements = vec![];
@ -407,6 +425,40 @@ impl Processor {
_ => unimplemented!("ran into unhandled rule: {:?}", line.as_span())
}
},
Rule::shader_struct_def => {
let mut struct_inner = line.clone().into_inner();
let struct_name = struct_inner.next().unwrap();
let mut requirements = vec![];
while let (Some(ident), Some(field_ty)) = (struct_inner.next(), struct_inner.next()) {
//panic!("got {} of type {}", ident.as_str(), field_ty.as_str());
let mut ty_inner = field_ty.into_inner();
let ty_inner = ty_inner.next().unwrap();
match ty_inner.as_rule() {
Rule::shader_external_variable | Rule::shader_type => {
//requirements.push(value)
if let Some(req) = get_external_var_requirement(&module, ty_inner) {
requirements.push(req);
}
},
_ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span())
}
}
let sname = struct_name.as_str().to_string();
let line_span = line.as_span();
let start_pos = line_span.start();
let end_pos = line_span.end();
module.structs.insert(sname.clone(), Definition {
name: sname,
requirements,
start_pos,
end_pos,
});
},
Rule::cws => (),
Rule::newline => (),
_ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span())
@ -466,7 +518,8 @@ impl Processor {
let module = self.modules.get(module_path).unwrap();
let mut output = String::new();
compile_definitions(&self.modules, module, &mut output)?;
let mut compiled = HashSet::new();
compile_definitions(&self.modules, module, &mut compiled, &mut output)?;
Ok(output)
}
@ -578,8 +631,14 @@ impl Processor {
}
fn try_find_requirement_module(module: &Module, req_name: &str) -> Option<String> {
//debug!("imports: {:?}", module.item_imports.values());
//debug!("Module: {}, req: {}", module.name, req_name);
for import in module.item_imports.values() {
//debug!("Import: {:?}", import);
if import.imports.contains(&req_name.to_string()) {
//debug!("has it");
return Some(import.module.clone());
}
}
@ -587,35 +646,72 @@ fn try_find_requirement_module(module: &Module, req_name: &str) -> Option<String
None
}
#[instrument(fields(module = module.name), skip_all)]
fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, output: &mut String) -> Result<(), PreprocessorError> {
for (_, funcs) in &module.functions {
//#[instrument(fields(module = module.name, require=tracing::field::Empty), skip_all)]
fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, compiled_modules: &mut HashSet<String>, output: &mut String) -> Result<(), PreprocessorError> {
let e = debug_span!("compile_definitions", module = module.name, require = tracing::field::Empty).entered();
let defs = module.functions.iter().chain(module.vars.iter()).chain(module.structs.iter());
for (_, funcs) in defs { //&module.functions {
let mut requirements = VecDeque::from(funcs.requirements.clone());
while let Some(mut req) = requirements.pop_front() {
e.record("require", req.name.clone());
if req.module.is_none() {
let mod_name = try_find_requirement_module(&module, &req.name);
req.module = mod_name;
debug!("req module: {:?}", req.module);
}
if let Some(module_name) = &req.module {
let req_module = modules.get(module_name)
.unwrap_or_else(|| panic!("invalid module import: {}", module_name));
// get the definition from the module that defines the import
let req_def = req_module.functions.get(&req.name)
.or_else(|| req_module.vars.get(&req.name))
.or_else(|| req_module.structs.get(&req.name))
.unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name));
if !req_def.requirements.is_empty() {
let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
debug!("Found requirement: {}, with the following sub-requirements: {:?}", req_def.name, sub_req_names);
for sub in &req_def.requirements {
let sub_module = sub.module.clone().or_else(|| try_find_requirement_module(&module, &req.name));
if let Some(sub_mod) = &sub_module {
if !compiled_modules.contains(sub_mod) {
//panic!("NEED TO COMPILE");
compiled_modules.insert(sub_mod.clone());
let sub_mod = modules.get(sub_mod).expect("unknown module from import");
debug!("found sub requirement module: {}", sub_mod.name);
let module_name = &sub_mod.name;
/* let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
debug!("Found requirement: {}, with the following sub-requirements: {:?} from {}", req_def.name, sub_req_names, req_module.name); */
let mut requirements_output = String::new();
compile_definitions(modules, sub_mod, compiled_modules, &mut requirements_output)?;
output.write_fmt(format_args!("\n// REQUIREMENTS OF {}::{}\n", module_name, req.name))?;
output.push_str(&requirements_output);
output.push_str("\n");
} else {
debug!("Requirement module is already compiled, skipping...");
}
} else {
debug!("Requirement has no module, assuming its local...");
}
}
/* let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
debug!("Found requirement: {}, with the following sub-requirements: {:?} from {}", req_def.name, sub_req_names, req_module.name);
let mut requirements_output = String::new();
compile_definitions(modules, req_module, &mut requirements_output)?;
output.write_fmt(format_args!("\n// REQUIREMENTS OF {}::{}\n", module_name, req.name))?;
output.push_str(&requirements_output);
output.push_str("\n");
output.push_str("\n"); */
}
let func_src = &req_module.src[req_def.start_pos..req_def.end_pos];

2
src/wgsl.pest
View File

@ -41,7 +41,7 @@ shader_struct_def = {
"struct" ~ ws ~ shader_ident ~ ws* ~ "{" ~ NEWLINE ~
ws* ~ shader_ident ~ shader_var_type ~
("," ~ NEWLINE ~ ws* ~ shader_ident ~ shader_var_type)* ~
("," ~ NEWLINE)? ~
","? ~ NEWLINE? ~
"}"
}
shader_type_alias_def = { "alias" ~ ws ~ shader_ident ~ ws* ~ "=" ~ ws* ~ (shader_external_variable | shader_type) ~ ";" }