.vscode/launch.json

@@ -7,15 +7,15 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug wgsl_preprocessor",
+            "name": "Debug shader_prepoc",
             "cargo": {
                 "args": [
                     "build",
                     //"--manifest-path", "${workspaceFolder}/examples/testbed/Cargo.toml"
-                    "--bin=wgsl_preprocessor",
+                    "--bin=shader_prepoc",
                 ],
                 "filter": {
-                    "name": "wgsl_preprocessor",
+                    "name": "shader_prepoc",
                     "kind": "bin"
                 }
             },
@@ -25,55 +25,33 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'wgsl_preprocessor'",
+            "name": "Debug unit tests in executable 'shader_prepoc'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=wgsl_preprocessor",
+                    "--bin=shader_prepoc",
-                    "--package=wgsl_preprocessor"
+                    "--package=shader_prepoc"
                 ],
                 "filter": {
-                    "name": "wgsl_preprocessor",
+                    "name": "shader_prepoc",
                     "kind": "bin"
                 }
             },
             "args": [],
             "cwd": "${workspaceFolder}"
         },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug specific test in executable 'wgsl_preprocessor'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=wgsl_preprocessor",
-                    "tests::import_struct_with_imported_fields",
-                    "--",
-                    "--exact --nocapture"
-                ],
-                "filter": {
-                    "name": "wgsl_preprocessor",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
         /* {
             "type": "lldb",
             "request": "launch",
-            "name": "Test wgsl_preprocessor",
+            "name": "Test shader_prepoc",
             "cargo": {
                 "args": [
                     "test",
-                    "--bin=wgsl_preprocessor",
+                    "--bin=shader_prepoc",
                 ],
                 "filter": {
-                    "name": "wgsl_preprocessor",
+                    "name": "shader_prepoc",
                     "kind": "bin"
                 }
             },

src/lib.rs

@@ -19,13 +19,13 @@ pub enum ExternalUsageType {
     Function,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct Import {
     module: String,
     imports: Vec<String>,
 }
 
-#[derive(Clone, PartialEq, Eq, Hash)]
+#[derive(Clone)]
 pub struct DefRequirement {
     /// None if the requirement is local
     module: Option<String>,
@@ -34,7 +34,6 @@ pub struct DefRequirement {
 
 #[derive(Clone)]
 pub struct Definition {
-    #[allow(dead_code)]
     name: String,
     requirements: Vec<DefRequirement>,
     /// The start byte position as a `usize`.
@@ -51,8 +50,6 @@ pub struct Module {
     src: String,
     /// Constants that this module defines
     pub vars: HashMap<String, Definition>,
-    pub structs: HashMap<String, Definition>,
-    pub aliases: HashMap<String, Definition>,
     /// Functions that this module defines
     pub functions: HashMap<String, Definition>,
     /// Imports of things per module
@@ -64,16 +61,6 @@ pub struct Module {
     module_imports: HashSet<String>,
 }
 
-impl Module {
-    /// Get a definition of any type from this module.
-    pub fn get_definition(&self, name: &str) -> Option<&Definition> {
-        self.functions.get(name)
-            .or_else(|| self.vars.get(name))
-            .or_else(|| self.structs.get(name))
-            .or_else(|| self.aliases.get(name))
-    }
-}
-
 /// Recursively find files in `path`.
 pub(crate) fn recurse_files(path: impl AsRef<Path>) -> std::io::Result<Vec<PathBuf>> {
     let mut buf = vec![];
@@ -150,12 +137,6 @@ fn main() -> vec4<f32> {
     /// i.e., `simple::some_const`.
     #[test]
     fn double_layer_import_indirect_imports() {
-        /* tracing_subscriber::fmt()
-            // enable everything
-            .with_max_level(tracing::Level::TRACE)
-            // sets this to be the default, global collector for this application.
-            .init(); */
-
         let mut p = Processor::new();
         p.parse_module(&INNER_MODULE).unwrap()
             .expect("failed to find module");
@@ -169,7 +150,7 @@ fn main() -> vec4<f32> {
         assert!(out.contains("const simple_scalar"), "definition of imported `const simple_scalar` is missing!");
         assert!(out.contains("fn do_something_cool("), "definition of imported `fn do_something_cool` is missing!");
 
-        assert!(out.contains("const scalar"), "definition of imported dependency, `const scalar` is missing! {}", out);
+        assert!(out.contains("const scalar"), "definition of imported dependency, `const scalar` is missing!");
         assert!(out.contains("fn mult_some_nums("), "definition of imported dependency, `fn mult_some_nums` is missing!");
 
         assert!(out.contains("fn do_something_cool("), "definition of imported `fn do_something_cool` is missing!");
@@ -199,11 +180,11 @@ fn main() -> vec4<f32> {
     return vec4<f32>(1.0);
 }"#;
 
-        /* tracing_subscriber::fmt()
+        tracing_subscriber::fmt()
             // enable everything
             .with_max_level(tracing::Level::TRACE)
             // sets this to be the default, global collector for this application.
-            .init(); */
+            .init();
 
         let mut p = Processor::new();
         p.parse_module(BINDINGS_MODULE).unwrap()
@@ -240,11 +221,11 @@ fn main() -> vec4<f32> {
     return vec4<f32>(1.0);
 }"#;
 
-        /* tracing_subscriber::fmt()
+        tracing_subscriber::fmt()
             // enable everything
             .with_max_level(tracing::Level::TRACE)
             // sets this to be the default, global collector for this application.
-            .init(); */
+            .init();
 
         let mut p = Processor::new();
         p.parse_module(BINDINGS_MODULE).unwrap()
@@ -290,11 +271,11 @@ fn main() -> vec4<f32> {
     return vec4<f32>(1.0);
 }"#;
 
-        /* tracing_subscriber::fmt()
+        tracing_subscriber::fmt()
             // enable everything
             .with_max_level(tracing::Level::TRACE)
             // sets this to be the default, global collector for this application.
-            .init(); */
+            .init();
 
         let mut p = Processor::new();
         p.parse_module(BINDINGS_MODULE).unwrap()
@@ -312,171 +293,4 @@ fn main() -> vec4<f32> {
         assert!(out.contains("var<storage, read> light_indices"), "missing light_indices binding: {}", out);
         assert!(out.contains("var<storage, read_write> some_buffer"), "missing some_buffer binding: {}", out);
     }
-
+}
-    #[test]
-    fn import_struct() {
-        const BINDINGS_MODULE: &'static str =
-r#"#define_module engine::lights
-
-struct Light {
-    intensity: f32
-}"#;
-
-        const IMPORTING_MODULE: &'static str =
-r#"#define_module engine::main
-#import engine::lights::{Light}
-
-fn main() -> vec4<f32> {
-    // imports must be used to be generated
-    let base_light = Light(1.0);
-
-    return vec4<f32>(1.0);
-}"#;
-
-        /* tracing_subscriber::fmt()
-            // enable everything
-            .with_max_level(tracing::Level::TRACE)
-            // sets this to be the default, global collector for this application.
-            .init(); */
-
-        let mut p = Processor::new();
-        p.parse_module(BINDINGS_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
-            .expect("failed to find main module def");
-
-        let lights_mod = p.modules.get("engine::lights").unwrap();
-        assert!(lights_mod.structs.contains_key("Light"), "`Light` struct definition was not parsed");
-
-        let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
-        assert!(out.contains("struct Light"), "missing `Light` struct definition: {}", out);
-    }
-
-    #[test]
-    fn import_struct_with_imported_fields() {
-        const TYPE_MODULE: &'static str =
-r#"#define_module engine::types
-
-struct Something {
-    val: f32
-}"#;
-
-        const BINDINGS_MODULE: &'static str =
-r#"#define_module engine::lights
-#import engine::types::{Something}
-
-struct Light {
-    some: Something,
-    intensity: f32
-}"#;
-
-        const IMPORTING_MODULE: &'static str =
-r#"#define_module engine::main
-#import engine::lights::{Light}
-
-fn main() -> vec4<f32> {
-    // imports must be used to be generated
-    let base_light = Light(1.0);
-
-    return vec4<f32>(1.0);
-}"#;
-
-        /* tracing_subscriber::fmt()
-            // enable everything
-            .with_max_level(tracing::Level::TRACE)
-            // sets this to be the default, global collector for this application.
-            .init(); */
-
-        let mut p = Processor::new();
-        p.parse_module(TYPE_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        p.parse_module(BINDINGS_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
-            .expect("failed to find main module def");
-
-        let lights_mod = p.modules.get("engine::lights").unwrap();
-        assert!(lights_mod.structs.contains_key("Light"), "`Light` struct definition was not parsed");
-
-        let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
-        assert!(out.contains("struct Light"), "missing `Light` struct definition: {}", out);
-        assert!(out.contains("struct Something"), "missing `Something` struct definition: {}", out);
-    }
-
-    #[test]
-    fn import_type_alias() {
-        const TYPE_MODULE: &'static str =
-r#"#define_module engine::types
-
-alias LightIntensity = f32;"#;
-
-        const IMPORTING_MODULE: &'static str =
-r#"#define_module engine::main
-#import engine::types::{LightIntensity}
-
-fn main() -> vec4<f32> {
-    // imports must be used to be generated
-    let intensity: LightIntensity = 1.0;
-
-    return vec4<f32>(1.0);
-}"#;
-
-        /* tracing_subscriber::fmt()
-            // enable everything
-            .with_max_level(tracing::Level::TRACE)
-            // sets this to be the default, global collector for this application.
-            .init(); */
-
-        let mut p = Processor::new();
-        p.parse_module(TYPE_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
-            .expect("failed to find main module def");
-
-        let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
-        assert!(out.contains("alias LightIntensity"), "missing `LightIntensity` type alias: {}", out);
-    }
-
-    #[test]
-    fn import_type_alias_with_imports() {
-        const TYPE_MODULE: &'static str =
-r#"#define_module engine::types
-
-alias Number = f32;"#;
-
-        const LIGHT_MODULE: &'static str =
-r#"#define_module engine::light
-#import engine::types::{Number}
-
-alias LightIntensity = Number;"#;
-
-        const IMPORTING_MODULE: &'static str =
-r#"#define_module engine::main
-#import engine::light::{LightIntensity}
-
-fn main() -> vec4<f32> {
-    // imports must be used to be generated
-    let intensity: LightIntensity = 1.0;
-
-    return vec4<f32>(1.0);
-}"#;
-
-        /* tracing_subscriber::fmt()
-            // enable everything
-            .with_max_level(tracing::Level::TRACE)
-            // sets this to be the default, global collector for this application.
-            .init(); */
-
-        let mut p = Processor::new();
-        p.parse_module(TYPE_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        p.parse_module(LIGHT_MODULE).unwrap()
-            .expect("failed to find bindings module def");
-        let importing_mod = p.parse_module(IMPORTING_MODULE).unwrap()
-            .expect("failed to find main module def");
-
-        let out = p.process_file(&importing_mod, &IMPORTING_MODULE).unwrap();
-        assert!(out.contains("alias LightIntensity"), "missing `LightIntensity` type alias: {}", out);
-        assert!(out.contains("alias Number"), "missing `Number` type alias: {}", out);
-    }
-}

src/preprocessor.rs

@@ -63,46 +63,6 @@ pub enum PreprocessorError {
     ConflictingImport { from_module: String, name: String },
 }
 
-fn get_external_var_requirement(module: &Module, pair: Pair<Rule>) -> Option<DefRequirement> {
-    let var_name = pair.as_str();
-
-    if pair.as_rule() == Rule::shader_external_variable {
-        let (module_name, name) = var_name.rsplit_once("::").unwrap();
-
-        if RESERVED_WORDS.contains(&name) {
-            return None;
-        }
-
-        // Find the module that this variable is from.
-        // Starts by checking if the full module path was used.
-        // If it was not, then find the full path of the module.
-        let module_full_name = if module.module_imports.contains(module_name) {
-            Some(module_name.to_string())
-        } else {
-            module.module_imports.iter()
-                .find(|m| {
-                    m.ends_with(module_name)
-                }).cloned()
-        };
-
-        debug!("Binding module is {:?}", module_full_name);
-
-        Some(DefRequirement {
-            module: module_full_name,
-            name: name.to_string(),
-        })
-    } else { // The only possibility is `Rule::shader_type`
-        if RESERVED_WORDS.contains(&var_name) {
-            return None;
-        }
-
-        Some(DefRequirement {
-            module: None,
-            name: var_name.to_string(),
-        })
-    }
-}
-
 #[derive(Default)]
 pub struct Processor {
     pub modules: HashMap<String, Module>,
@@ -113,6 +73,28 @@ impl Processor {
         Self::default()
     }
 
+    // 
+    /* fn add_func_requirements(&self, found_requirements: &mut HashSet<String>, requirements: &mut Vec<DefRequirement>, fn_name: &str) {
+        if found_requirements.contains(fn_name) {
+            return;
+        }
+        found_requirements.insert(fn_name.to_string());
+
+        debug!("Found call to `{}`", fn_name);
+
+        // ignore reserved words
+        if RESERVED_WORDS.contains(&fn_name) {
+            return;
+        }
+
+        let req = DefRequirement {
+            // module is discovered later
+            module: None,
+            name: fn_name.to_string(),
+        };
+        requirements.push(req);
+    } */
+
     #[instrument(fields(module = module.name), skip_all)]
     fn get_imports_in_block(&mut self, module: &mut Module, block: Pair<Rule>, found_requirements: &mut HashSet<String>) -> Vec<DefRequirement> {
         let mut requirements = vec![];
@@ -425,67 +407,6 @@ impl Processor {
                                     _ => unimplemented!("ran into unhandled rule: {:?}", line.as_span())
                                 }
                             },
-                            Rule::shader_struct_def => {
-                                let mut struct_inner = line.clone().into_inner();
-                                let struct_name = struct_inner.next().unwrap();
-
-                                // iterate through fields in struct
-                                let mut requirements = vec![];
-                                while let (Some(_), Some(field_ty)) = (struct_inner.next(), struct_inner.next()) {
-                                    //panic!("got {} of type {}", ident.as_str(), field_ty.as_str());
-                                    let mut ty_inner = field_ty.into_inner();
-                                    let ty_inner = ty_inner.next().unwrap();
-
-                                    match ty_inner.as_rule() {
-                                        Rule::shader_external_variable | Rule::shader_type => {
-                                            //requirements.push(value)
-                                            if let Some(req) = get_external_var_requirement(&module, ty_inner) {
-                                                requirements.push(req);
-                                            }
-                                        },
-                                        _ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span())
-                                    }
-                                }
-
-                                let sname = struct_name.as_str().to_string();
-                                let line_span = line.as_span();
-                                let start_pos = line_span.start();
-                                let end_pos = line_span.end();
-
-                                module.structs.insert(sname.clone(), Definition {
-                                    name: sname,
-                                    requirements,
-                                    start_pos,
-                                    end_pos,
-                                });
-                            },
-                            Rule::shader_type_alias_def => {
-                                let mut inner = line.clone().into_inner();
-
-                                // get rules
-                                let alias_name = inner.next().unwrap();
-                                let alias_to = inner.next().unwrap();
-
-                                // get the requirements of the alias if there are any
-                                let requirements = if let Some(req) = get_external_var_requirement(&module, alias_to) {
-                                    debug!("Type req: {}", req.name);
-                                    vec![req]
-                                } else { vec![] };
-
-                                let aname = alias_name.as_str().to_string();
-                                debug!("Type alias `{}`", aname);
-
-                                let line_span = line.as_span();
-                                let start_pos = line_span.start();
-                                let end_pos = line_span.end();
-
-                                module.aliases.insert(aname.clone(), Definition {
-                                    name: aname,
-                                    requirements,
-                                    start_pos,
-                                    end_pos,
-                                });
-                            },
                             Rule::cws => (),
                             Rule::newline => (),
                             _ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span())
@@ -545,8 +466,7 @@ impl Processor {
         let module = self.modules.get(module_path).unwrap();
 
         let mut output = String::new();
-        let mut compiled = HashSet::new();
+        compile_definitions(&self.modules, module, &mut output)?;
-        compile_definitions(&self.modules, module, &mut compiled, &mut output)?;
         Ok(output)
     }
 
@@ -658,14 +578,8 @@ impl Processor {
 }
 
 fn try_find_requirement_module(module: &Module, req_name: &str) -> Option<String> {
-    debug!("looking for '{}' in module '{}' that imports: {:?}", req_name, module.name, module.item_imports.values());
-    //debug!("Module: {}, req: {}", module.name, req_name);
-
     for import in module.item_imports.values() {
-        //debug!("Import: {:?}", import);
-
         if import.imports.contains(&req_name.to_string()) {
-            //debug!("has it");
             return Some(import.module.clone());
         }
     }
@@ -673,81 +587,35 @@ fn try_find_requirement_module(module: &Module, req_name: &str) -> Option<String
     None
 }
 
-//#[instrument(fields(module = module.name, require=tracing::field::Empty), skip_all)]
+#[instrument(fields(module = module.name), skip_all)]
-fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, compiled_definitions: &mut HashSet<DefRequirement>, output: &mut String) -> Result<(), PreprocessorError> {
+fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, output: &mut String) -> Result<(), PreprocessorError> {
-    let e = debug_span!("compile_definitions", module = module.name, require = tracing::field::Empty, sub_require = tracing::field::Empty).entered();
+    for (_, funcs) in &module.functions {
-
-    let defs = module.functions.iter().chain(module.vars.iter()).chain(module.structs.iter()).chain(module.aliases.iter());
-    for (_, funcs) in defs { //&module.functions {
         let mut requirements = VecDeque::from(funcs.requirements.clone());
         
         while let Some(mut req) = requirements.pop_front() {
-            e.record("require", req.name.clone());
-            
             if req.module.is_none() {
                 let mod_name = try_find_requirement_module(&module, &req.name);
                 req.module = mod_name;
-                debug!("req module: {:?}", req.module);
             }
 
             if let Some(module_name) = &req.module {
                 let req_module = modules.get(module_name)
                     .unwrap_or_else(|| panic!("invalid module import: {}", module_name));
                 
-                // get the definition from the module that defines the import
+                let req_def = req_module.functions.get(&req.name)
-                let req_def = req_module.get_definition(&req.name)
+                    .or_else(|| req_module.vars.get(&req.name))
                     .unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name));
 
                 if !req_def.requirements.is_empty() {
-                    for mut sub in req_def.requirements.clone() {
+                    let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
-                        e.record("sub_require", sub.name.clone());
+                    debug!("Found requirement: {}, with the following sub-requirements: {:?}", req_def.name, sub_req_names);
-
-                        println!("searching for sub requirement module in module: '{}'", req_module.name);
-
-                        if sub.module.is_none() {
-                            sub.module = try_find_requirement_module(&req_module, &sub.name);
-                        }
-
-                        if let Some(sub_mod) = &sub.module {
-                            if !compiled_definitions.contains(&sub) {
-                                compiled_definitions.insert(sub.clone());
-
-                                let sub_mod = modules.get(sub_mod).expect("unknown module from import");
-                                debug!("found sub requirement module: {}", sub_mod.name);
-                                let module_name = &sub_mod.name;
-
-                                let mut requirements_output = String::new();
-                                compile_definitions(modules, sub_mod, compiled_definitions, &mut requirements_output)?;
-                                
-                                output.write_fmt(format_args!("\n// REQUIREMENTS OF {}::{}\n", module_name, req.name))?;
-                                output.push_str(&requirements_output);
-                                output.push_str("\n");
-
-                                // find the definition of the sub requirement
-                                let sub_def = sub_mod.get_definition(&sub.name)
-                                    .unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name));
-
-                                // write the source of the sub requirement to the output
-                                let req_src = &sub_mod.src[sub_def.start_pos..sub_def.end_pos];
-                                output.write_fmt(format_args!("// SOURCE {}::{}\n", module_name, sub.name))?;
-                                output.push_str(req_src);
-                                output.push_str("\n");
-                            } else {
-                                debug!("Requirement module is already compiled, skipping...");
-                            }
-                        } else {
-                            debug!("Requirement has no module, assuming its local...");
-                        }
-                    }
-                    /* let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
-                    debug!("Found requirement: {}, with the following sub-requirements: {:?} from {}", req_def.name, sub_req_names, req_module.name);
 
                     let mut requirements_output = String::new();
                     compile_definitions(modules, req_module, &mut requirements_output)?;
                     
                     output.write_fmt(format_args!("\n// REQUIREMENTS OF {}::{}\n", module_name, req.name))?;
                     output.push_str(&requirements_output);
-                    output.push_str("\n"); */
+                    output.push_str("\n");
                 }
 
                 let func_src = &req_module.src[req_def.start_pos..req_def.end_pos];

src/wgsl.pest

@@ -37,14 +37,6 @@ shader_value = { shader_value_bool | shader_value_num  }
 
 shader_var_type = { ":" ~ ws* ~ (shader_external_variable | shader_type) }
 shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" }
-shader_type_alias_def = { ("alias" | "type") ~ ws ~ shader_ident ~ ws* ~ "=" ~ ws* ~ (shader_external_variable | shader_type) ~ ";" }
-shader_struct_def = {
-    "struct" ~ ws ~ shader_ident ~ ws* ~ "{" ~ NEWLINE ~
-        ws* ~ shader_ident ~ shader_var_type ~
-        ("," ~ NEWLINE ~ ws* ~ shader_ident ~ shader_var_type)* ~
-        ","? ~ NEWLINE? ~ 
-    "}"
-}
 
 shader_group_binding = { "@group(" ~ NUMBER ~ ") @binding(" ~ NUMBER ~ ")" }
 shader_binding_var_constraint = {
@@ -66,10 +58,8 @@ shader_fn_def = {
 // a line of shader code, including white space
 shader_code_line = {
     shader_fn_def |
-    (shader_struct_def ~ newline?) |
     (shader_const_def ~ newline?) |
     (shader_binding_def ~ newline?) |
-    (shader_type_alias_def ~ newline?) |
     ws* ~ newline
 }
 //shader_code_line = { shader_fn_def | shader_const_def | (ws* ~ (shader_external_code | shader_code)* ~ ws*) }