From 7940cbdba9b1acca5b77bdbeb53e188ea0c52c44 Mon Sep 17 00:00:00 2001 From: SeanOMik Date: Sat, 3 Aug 2024 22:24:55 -0400 Subject: [PATCH] remove comments from modules before parsing, use tracing for logging --- Cargo.lock | 156 +++++++++++++++++++-- Cargo.toml | 3 +- shaders/base.wgsl | 4 +- shaders/inner_include.wgsl | 10 ++ shaders/simple.wgsl | 2 + src/compiler.rs | 35 ----- src/main.rs | 25 +--- src/preprocessor.rs | 277 ++++++++++++++++++++++++------------- src/wgsl.pest | 3 +- 9 files changed, 346 insertions(+), 169 deletions(-) delete mode 100644 src/compiler.rs diff --git a/Cargo.lock b/Cargo.lock index 3f4a3cc..2cdd055 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,12 +46,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - [[package]] name = "generic-array" version = "0.14.7" @@ -63,13 +57,10 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.13.0" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" @@ -77,18 +68,40 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "pest" version = "2.7.11" @@ -134,6 +147,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + [[package]] name = "proc-macro2" version = "1.0.86" @@ -167,12 +186,28 @@ dependencies = [ name = "shader_prepoc" version = "0.1.0" dependencies = [ - "itertools", "pest", "pest_derive", "thiserror", + "tracing", + "tracing-subscriber", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "syn" version = "2.0.72" @@ -204,6 +239,73 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + [[package]] name = "typenum" version = "1.17.0" @@ -222,8 +324,36 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml index 40dbee2..1abd5c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,8 @@ version = "0.1.0" edition = "2021" [dependencies] -itertools = "0.13.0" pest = "2.7.11" pest_derive = "2.7.11" thiserror = "1.0.63" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/shaders/base.wgsl b/shaders/base.wgsl index 746eb84..655fcc9 100644 --- a/shaders/base.wgsl +++ b/shaders/base.wgsl @@ -1,7 +1,7 @@ #define_module base -#import simple::{do_something_cool} +#import simple fn main() -> vec4 { - let a = do_something_cool(10.0); + let a = simple::simple_scalar * simple::do_something_cool(10.0); return vec4(vec3(a), 1.0); } \ No newline at end of file diff --git a/shaders/inner_include.wgsl b/shaders/inner_include.wgsl index 35183d7..c860750 100644 --- a/shaders/inner_include.wgsl +++ b/shaders/inner_include.wgsl @@ -2,6 +2,16 @@ const scalar: f32 = 5.0; +// test to ignore comments + +/* + some test ig + + /* inner comment */ +*/ + +/* c-style comment */ + fn mult_some_nums(a: f32, b: f32) -> f32 { let c = a * b; return c; diff --git a/shaders/simple.wgsl b/shaders/simple.wgsl index ac5fbfc..6bb8c34 100644 --- a/shaders/simple.wgsl +++ b/shaders/simple.wgsl @@ -1,6 +1,8 @@ #define_module simple #import inner::some_include::{scalar, mult_some_nums} +const simple_scalar: f32 = 50.0; + fn do_something_cool(in: f32) -> f32 { return scalar * mult_some_nums(in, 2.0); } \ No newline at end of file diff --git a/src/compiler.rs b/src/compiler.rs deleted file mode 100644 index 7595ab9..0000000 --- a/src/compiler.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::collections::HashMap; - -use crate::{Definition, Module, PreprocessorError, Processor}; - -/// Compile a module including its imports into a single module. -#[derive(Default)] -pub struct Compiler { - preprocessor: Processor, -} - -impl Compiler { - /// Add a module to the compiler - /// - /// Returns `None` if the module does not define an include identifier. - pub fn add_module(&mut self, module_src: &str) -> Result, PreprocessorError> { - self.preprocessor.parse_module(module_src) - } - - pub fn compile_module(self, module_src: &str) -> Result { - todo!() - } -} - -/* pub struct Source { - -} */ - -pub struct ExpandableModule { - /// The name of the module. - name: String, - /// Constants that this module defines - pub constants: HashMap, - /// Functions that this module defines - pub functions: HashMap, -} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index b15c15e..9b1901d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,28 +1,24 @@ use std::{ collections::{HashMap, HashSet}, - fmt::Write, fs, path::{Path, PathBuf}, }; -use itertools::Itertools; -use pest::Parser; use pest_derive::Parser; mod preprocessor; pub use preprocessor::*; -mod compiler; -pub use compiler::*; - #[derive(Parser)] #[grammar = "wgsl.pest"] pub(crate) struct WgslParser; fn main() { - /* let mut successful_parse = WgslParser::parse(Rule::command_line, "#define_module inner::some_include").unwrap(); - let p = successful_parse.next().unwrap(); - println!("test: {}", p.as_str()); */ + tracing_subscriber::fmt() + // enable everything + .with_max_level(tracing::Level::TRACE) + // sets this to be the default, global collector for this application. + .init(); let mut p = Processor::new(); //let f = p.parse_modules("shaders", ["wgsl"]).unwrap(); @@ -78,7 +74,7 @@ fn main() { for (module, usages) in &module.import_usages { println!(" {}:", module); - for import in &usages.imports { + for import in usages.iter() { println!( " {:?} `{}` at {}:", import.ty, import.name, import.start_pos @@ -106,12 +102,6 @@ pub struct ExternalUsage { start_pos: usize, } -#[derive(Clone)] -pub struct ImportUsage { - module: String, - imports: Vec, -} - #[derive(Clone)] pub struct Import { module: String, @@ -123,7 +113,6 @@ pub struct DefRequirement { /// None if the requirement is local module: Option, name: String, - ty: ExternalUsageType, } #[derive(Clone)] @@ -151,7 +140,7 @@ pub struct Module { item_imports: HashMap, /// usages of imported things /// ie `other_module::scalar` - import_usages: HashMap, + import_usages: HashMap>, /// Imports of modules /// /// These modules are used along side `import_usages` diff --git a/src/preprocessor.rs b/src/preprocessor.rs index c5e3a3e..e969731 100644 --- a/src/preprocessor.rs +++ b/src/preprocessor.rs @@ -1,10 +1,11 @@ -use std::{collections::{HashMap, HashSet, VecDeque}, fmt::Write, fs, path::{Path, PathBuf}}; +use std::{collections::{HashMap, HashSet, VecDeque}, fmt::Write, fs, path::Path}; use pest::{iterators::Pair, Parser}; +use tracing::{debug, debug_span, instrument}; -use crate::{recurse_files, DefRequirement, Definition, ExternalUsageType, Import, Module, Rule, WgslParser}; +use crate::{recurse_files, DefRequirement, Definition, Import, Module, Rule, WgslParser}; -const RESERVED_WORDS: [&str; 171] = [ +const RESERVED_WORDS: [&str; 201] = [ "NULL", "Self", "abstract", "active", "alignas", "alignof", "as", "asm", "asm_fragment", "async", "attribute", "auto", "await", "become", "binding_array", "cast", "catch", "class", "co_await", "co_return", "co_yield", "coherent", "column_major", "common", "compile", @@ -25,7 +26,24 @@ const RESERVED_WORDS: [&str; 171] = [ "where", "with", "writeonly", "yield", "alias", "break", "case", "const", "const_assert", "continue", "continuing", "default", "diagnostic", "discard", "else", "enable", "false", "fn", "for", "if", "let", "loop", "override", "requires", "return", "struct", "switch", "true", - "var", "while" + "var", "while", + + // types + "vec4", "vec3", "vec2", + "mat2x2", "mat2x3", "mat2x4", + "mat3x2", "mat3x3", "mat3x4", + "mat4x2", "mat4x3", "mat4x4", + "f16", "f32", "i32", "u32", + "bool", "array", "atomic", + + // texture types + "texture_1d", + "texture_2d", "texture_2d_array", + "texture_depth_2d", "texture_depth_2d_array", + "texture_depth_cube", "texture_depth_cube_array", + "texture_3d", + "texture_cube_array", + "sampler", "sampler_comparison", ]; #[derive(Debug, thiserror::Error)] @@ -54,6 +72,28 @@ impl Processor { Self::default() } + fn add_func_requirements(&self, found_requirements: &mut HashSet, requirements: &mut Vec, fn_name: &str) { + if found_requirements.contains(fn_name) { + return; + } + found_requirements.insert(fn_name.to_string()); + + debug!("Found call to `{}`", fn_name); + + // ignore reserved words + if RESERVED_WORDS.contains(&fn_name) { + return; + } + + let req = DefRequirement { + // module is discovered later + module: None, + name: fn_name.to_string(), + }; + requirements.push(req); + } + + #[instrument(fields(module = module.name), skip_all)] fn get_imports_in_block(&mut self, module: &mut Module, block: Pair, found_requirements: &mut HashSet) -> Vec { let mut requirements = vec![]; @@ -66,48 +106,58 @@ impl Processor { Rule::shader_code_fn_usage => { let mut usage_inner = code.into_inner(); let fn_name = usage_inner.next().unwrap().as_str(); - let fn_args: Vec<&str> = usage_inner.map(|a| a.as_str()).collect(); - if found_requirements.contains(fn_name) { - continue; - } - found_requirements.insert(fn_name.to_string()); - - println!("Found call to {} with args: {:?}", fn_name, fn_args); - - // ignore reserved words - if RESERVED_WORDS.contains(&fn_name) { - continue; - } - - let req = DefRequirement { - // module is discovered later - module: None, - name: fn_name.to_string(), - ty: ExternalUsageType::Function, - }; - requirements.push(req); + self.add_func_requirements(found_requirements, &mut requirements, fn_name); }, Rule::shader_external_fn => { - let mut pairs = code.into_inner(); - // shader_external_variable is the only pair for this rule - let ident_name = pairs.next().unwrap().as_str().to_string(); + let mut usage_inner = code.into_inner(); + let fn_path = usage_inner.next().unwrap().as_str(); - println!("external fn: {ident_name}"); + if found_requirements.contains(fn_path) { + continue; + } + found_requirements.insert(fn_path.to_string()); + + let (fn_module, fn_name) = fn_path.rsplit_once("::").unwrap(); + + debug!("Found call to `{}::{}`", fn_module, fn_name); + + let req = DefRequirement { + module: Some(fn_module.into()), + name: fn_name.to_string(), + }; + requirements.push(req); }, Rule::shader_external_variable => { let pairs = code.into_inner(); // shader_external_variable is the only pair for this rule - let ident_name = pairs.as_str(); + let ident_path = pairs.as_str(); - println!("external var: {ident_name}"); + if found_requirements.contains(ident_path) { + continue; + } + found_requirements.insert(ident_path.to_string()); + + let (ident_module, ident_name) = ident_path.rsplit_once("::").unwrap(); + + debug!("Found call to `{}`", ident_name); + + // ignore reserved words + if RESERVED_WORDS.contains(&ident_name) { + continue; + } + + let req = DefRequirement { + module: Some(ident_module.into()), + name: ident_name.to_string(), + }; + requirements.push(req); }, Rule::newline => (), Rule::cws => (), Rule::shader_code_char => (), Rule::shader_ident => { let ident = code.as_str(); - println!("Found usage of ident: {}", ident); if found_requirements.contains(ident) { continue; @@ -123,7 +173,6 @@ impl Processor { // module is discovered later module: None, name: ident.to_string(), - ty: ExternalUsageType::Variable, }; requirements.push(req); }, @@ -138,14 +187,16 @@ impl Processor { /// Parse a module file to attempt to find the include identifier. /// /// Returns `None` if the module does not define an include identifier. + //#[instrument(skip(self, module_src))] pub fn parse_module( &mut self, module_src: &str, ) -> Result, PreprocessorError> { - //let unparsed_file = fs::read_to_string(path.as_ref())?; + //let current_span = Span::current(); + //let e = current_span.entered(); + let e = debug_span!("parse_module", module = tracing::field::Empty).entered(); - // add a new line to the end of the input to make the grammar happy - //let unparsed_file = format!("{unparsed_file}\n"); + let module_src = remove_comments(module_src)?; let file = WgslParser::parse(Rule::file, &module_src)? .next() @@ -171,7 +222,7 @@ impl Processor { let types: Vec = inner.map(|t| t.as_str().to_string()).collect(); - println!("found import of types from `{}`: `{:?}`", module_name, types); + debug!("Found imports from `{}`: `{:?}`", module_name, types); // add these type imports to imports of the module module.item_imports.entry(module_name.into()) @@ -184,16 +235,19 @@ impl Processor { Rule::import_module_command => { let mut inner = command_line.into_inner(); let module_name = inner.next().unwrap().as_str(); - println!("found import of module: {}", module_name); + debug!("Found module import: `{}`", module_name); module.module_imports.insert(module_name.into()); }, Rule::define_module_command => { let mut shader_file_pairs = command_line.into_inner(); - let shader_file = shader_file_pairs.next().unwrap(); - let shader_file = shader_file.as_str().to_string(); + let module_name = shader_file_pairs.next().unwrap(); + let module_name = module_name.as_str(); - module.name = shader_file; + e.record("module", module_name); + debug!("Defined module as `{}`", module_name); + + module.name = module_name.into(); } _ => unreachable!(), } @@ -205,7 +259,7 @@ impl Processor { let mut pairs = line.clone().into_inner(); // shader_ident is the only pair for this rule let fn_name = pairs.next().unwrap().as_str().to_string(); - println!("fn def: {fn_name:?}"); + debug!("Found function def: {fn_name}"); let fn_body = pairs.skip(2).next().unwrap(); let mut found_reqs = HashSet::default(); @@ -229,7 +283,7 @@ impl Processor { let mut pairs = line.clone().into_inner(); // shader_ident is the only pair for this rule let const_name = pairs.next().unwrap().as_str().to_string(); - println!("const def: {const_name:?}"); + debug!("Found const def: `{const_name}`"); let line_span = line.as_span(); let start_pos = line_span.start(); @@ -263,13 +317,10 @@ impl Processor { }, _ => unimplemented!("ran into unhandled rule: {:?}", line.as_span()) } - - - println!("code: {}", line.as_str()); } Rule::cws => (), Rule::newline => (), - _ => unimplemented!("ran into unhandled rule: {:?}", line.as_span()) + _ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span()) } } } @@ -297,6 +348,7 @@ impl Processor { /// * `path` - The path to search for files in. /// * `extensions` - The extensions that the discovered files must have. Make sure they have /// no leading '.' + #[instrument(skip(self, path, extensions))] pub fn parse_modules, const N: usize>( &mut self, path: P, @@ -320,35 +372,42 @@ impl Processor { Ok(parsed) } - fn generate_header(&mut self, module_path: &str) -> String { + #[instrument(skip(self))] + fn generate_header(&mut self, module_path: &str) -> Result { let module = self.modules.get(module_path).unwrap(); let mut output = String::new(); - compile_definitions(&self.modules, module, &mut output); - - output + compile_definitions(&self.modules, module, &mut output)?; + Ok(output) } + #[instrument(skip(self, shader_code_rule, output))] fn output_shader_code_line(&self, shader_code_rule: Pair, output: &mut String) -> Result<(), std::fmt::Error> { for line in shader_code_rule.into_inner() { - let (pos_line, pos_col) = line.line_col(); - match line.as_rule() { - Rule::shader_external_fn | Rule::shader_external_variable => { - let mut pairs = line.into_inner(); - // shader_external_variable is the only pair for this rule - let ident_name = pairs.next().unwrap().as_str().to_string(); + Rule::shader_external_fn => { + let mut pairs = line.clone().into_inner(); + let fn_path = pairs.next().unwrap().as_str().to_string(); + // the rest of the pairs are the arguments for the fn + let fn_args = pairs.as_str(); // remove the module from the identifier and write it to the output - if let Some((module_name, ident)) = ident_name.rsplit_once("::") { - output.write_str(ident)?; + if let Some((_, fn_name)) = fn_path.rsplit_once("::") { + output.write_str(fn_name)?; + output.write_fmt(format_args!("({})", fn_args))?; } else { // TODO: not really sure how this would get triggered - unimplemented!( - "this function is actually not external, i think" - ); + panic!("Unknown error, rule input: {}", line.as_str()); } }, + Rule::shader_external_variable => { + let pairs = line.clone().into_inner(); + // the last pair is the name of the variable + let ident_name = pairs.last().unwrap().as_str(); + + output.write_str(ident_name)?; + }, + //shader_external_variable Rule::shader_fn_def => { let mut rule_inner = line.into_inner(); @@ -381,7 +440,10 @@ impl Processor { Ok(()) } + #[instrument(skip(self, module_src))] pub fn process_file(&mut self, module_path: &str, module_src: &str) -> Result { + let module_src = remove_comments(module_src)?; + let mut out_string = String::new(); // the output will be at least the length of module_src out_string.reserve(module_src.len()); @@ -390,7 +452,7 @@ impl Processor { .next() .unwrap(); // get and unwrap the `file` rule; never fails - let header = self.generate_header(module_path); + let header = self.generate_header(module_path)?; out_string.write_str("// START OF IMPORT HEADER\n")?; out_string.write_str(&header)?; out_string.write_str("// END OF IMPORT HEADER\n")?; @@ -412,40 +474,6 @@ impl Processor { Rule::cws => (), Rule::shader_code_line => { self.output_shader_code_line(record, &mut out_string)?; - /* for line in record.into_inner() { - let (pos_line, pos_col) = line.line_col(); - - match line.as_rule() { - Rule::shader_external_fn | Rule::shader_external_variable => { - let mut pairs = line.into_inner(); - // shader_external_variable is the only pair for this rule - let ident_name = pairs.next().unwrap().as_str().to_string(); - - // remove the module from the identifier and write it to the output - if let Some((module_name, ident)) = ident_name.rsplit_once("::") { - out_string.write_str(ident)?; - } else { - // TODO: not really sure how this would get triggered - unimplemented!( - "this function is actually not external, i think" - ); - } - } - Rule::shader_code | Rule::shader_const_def => { - let input = line.as_str(); - out_string.write_str(&input)?; - }, - /* Rule::shader_fn_def => { - - }, */ - Rule::cws => { - let input = line.as_str(); - out_string.write_str(&input)?; - }, - Rule::newline => (), - _ => unimplemented!("ran into unhandled rule: {:?}", line.as_rule()), - } - } */ }, Rule::newline => { let input = record.as_str(); @@ -470,6 +498,7 @@ fn try_find_requirement_module(module: &Module, req_name: &str) -> Option, module: &Module, output: &mut String) -> Result<(), PreprocessorError> { for (_, funcs) in &module.functions { let mut requirements = VecDeque::from(funcs.requirements.clone()); @@ -488,10 +517,10 @@ fn compile_definitions(modules: &HashMap, module: &Module, outpu .or_else(|| req_module.constants.get(&req.name)) .unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name)); - let sub_req_names: Vec = req_def.requirements.iter().map(|r| r.name.clone()).collect(); - println!("got req: {}, subreqs: {:?}", req_def.name, sub_req_names); - if !req_def.requirements.is_empty() { + let sub_req_names: Vec = req_def.requirements.iter().map(|r| r.name.clone()).collect(); + debug!("Found requirement: {}, with the following sub-requirements: {:?}", req_def.name, sub_req_names); + let mut requirements_output = String::new(); compile_definitions(modules, req_module, &mut requirements_output)?; @@ -504,9 +533,59 @@ fn compile_definitions(modules: &HashMap, module: &Module, outpu output.write_fmt(format_args!("// SOURCE {}::{}\n", module_name, req.name))?; output.push_str(func_src); output.push_str("\n"); + } else { + debug!("Could not find module for `{}`, assuming its local", req.name); } } } Ok(()) +} + +#[instrument(skip(text))] +fn remove_comments(text: &str) -> Result { + let mut output = String::new(); + output.reserve(text.len()); + + let mut comment_layers = 0; + for line in text.lines() { + + if let Some(line_comment_start) = line.find("//") { + let uncommented = &line[..line_comment_start]; + output.write_str(uncommented)?; + continue; + } + + let (block_start, block_end) = (line.find("/*"), line.find("*/")); + if let (Some(start), Some(end)) = (block_start, block_end) { + if comment_layers == 0 { + let before = &line[..start]; + output.write_str(before)?; + + let after = &line[end + 2..]; + output.write_str(after)?; + } + } else if let Some(block_comment_start) = block_start { + if comment_layers == 0 { + let uncommented = &line[..block_comment_start]; + output.write_str(uncommented)?; + } + + comment_layers += 1; + } else if let Some(block_uncomment_start) = block_end { + if comment_layers == 1 { + let uncommented = &line[block_uncomment_start + 2..]; + output.write_str(uncommented)?; + } + + comment_layers -= 1; + } else if comment_layers == 0 { + output.write_str(line)?; + output.write_str("\n")?; + } + + + } + + Ok(output) } \ No newline at end of file diff --git a/src/wgsl.pest b/src/wgsl.pest index f75ca38..d7150b2 100644 --- a/src/wgsl.pest +++ b/src/wgsl.pest @@ -30,7 +30,8 @@ shader_code = { shader_code_block | shader_code_fn_usage | shader_value | shader // usages of code from another module shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ } -shader_external_fn = { shader_external_variable ~ "(" ~ ANY* ~ ")" } +//shader_fn_args2 = { shader_var_name_type? ~ (ws* ~ "," ~ ws* ~ shader_var_name_type)* } +shader_external_fn = { shader_external_variable ~ "(" ~ shader_code_fn_arg ~ ("," ~ ws* ~ shader_code_fn_arg)* ~ ")" } shader_external_code = _{ shader_external_fn | shader_external_variable } shader_actual_code_line = _{ shader_external_code | shader_code }