SeanOMik
/
wgsl-preprocessor
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

remove comments from modules before parsing, use tracing for logging

This commit is contained in:
SeanOMik 2024-08-03 22:24:55 -04:00
parent e85adce6e3
commit 7940cbdba9
9 changed files with 346 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
Cargo.lock generated
View File

@ -46,12 +46,6 @@ dependencies = [
"crypto-common", "crypto-common",
] ]
[[package]]
name = "either"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]] [[package]]
name = "generic-array" name = "generic-array"
version = "0.14.7" version = "0.14.7"
@ -63,13 +57,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "itertools" name = "lazy_static"
version = "0.13.0" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
dependencies = [
"either",
]
[[package]] [[package]]
name = "libc" name = "libc"
@ -77,18 +68,40 @@ version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "log"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.4" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.19.0" version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.7.11" version = "2.7.11"
@ -134,6 +147,12 @@ dependencies = [
"sha2", "sha2",
] ]
[[package]]
name = "pin-project-lite"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.86" version = "1.0.86"
@ -167,12 +186,28 @@ dependencies = [
name = "shader_prepoc" name = "shader_prepoc"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"itertools",
"pest", "pest",
"pest_derive", "pest_derive",
"thiserror", "thiserror",
"tracing",
"tracing-subscriber",
] ]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.72" version = "2.0.72"
@ -204,6 +239,73 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "thread_local"
version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "tracing"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"nu-ansi-term",
"sharded-slab",
"smallvec",
"thread_local",
"tracing-core",
"tracing-log",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.17.0" version = "1.17.0"
@ -222,8 +324,36 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.5" version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

3
Cargo.toml
View File

@ -4,7 +4,8 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
itertools = "0.13.0"
pest = "2.7.11" pest = "2.7.11"
pest_derive = "2.7.11" pest_derive = "2.7.11"
thiserror = "1.0.63" thiserror = "1.0.63"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"

4
shaders/base.wgsl
View File

@ -1,7 +1,7 @@
#define_module base #define_module base
#import simple::{do_something_cool} #import simple
fn main() -> vec4<f32> { fn main() -> vec4<f32> {
let a = do_something_cool(10.0); let a = simple::simple_scalar * simple::do_something_cool(10.0);
return vec4<f32>(vec3<f32>(a), 1.0); return vec4<f32>(vec3<f32>(a), 1.0);
} }

10
shaders/inner_include.wgsl
View File

@ -2,6 +2,16 @@
const scalar: f32 = 5.0; const scalar: f32 = 5.0;
// test to ignore comments
/*
some test ig
/* inner comment */
*/
/* c-style comment */
fn mult_some_nums(a: f32, b: f32) -> f32 { fn mult_some_nums(a: f32, b: f32) -> f32 {
let c = a * b; let c = a * b;
return c; return c;

2
shaders/simple.wgsl
View File

@ -1,6 +1,8 @@
#define_module simple #define_module simple
#import inner::some_include::{scalar, mult_some_nums} #import inner::some_include::{scalar, mult_some_nums}
const simple_scalar: f32 = 50.0;
fn do_something_cool(in: f32) -> f32 { fn do_something_cool(in: f32) -> f32 {
return scalar * mult_some_nums(in, 2.0); return scalar * mult_some_nums(in, 2.0);
} }

35
src/compiler.rs
View File

@ -1,35 +0,0 @@
use std::collections::HashMap;
use crate::{Definition, Module, PreprocessorError, Processor};
/// Compile a module including its imports into a single module.
#[derive(Default)]
pub struct Compiler {
preprocessor: Processor,
}
impl Compiler {
/// Add a module to the compiler
///
/// Returns `None` if the module does not define an include identifier.
pub fn add_module(&mut self, module_src: &str) -> Result<Option<String>, PreprocessorError> {
self.preprocessor.parse_module(module_src)
}
pub fn compile_module(self, module_src: &str) -> Result<String, PreprocessorError> {
todo!()
}
}
/* pub struct Source {
} */
pub struct ExpandableModule {
/// The name of the module.
name: String,
/// Constants that this module defines
pub constants: HashMap<String, String>,
/// Functions that this module defines
pub functions: HashMap<String, String>,
}

25
src/main.rs
View File

@ -1,28 +1,24 @@
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
fmt::Write,
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use itertools::Itertools;
use pest::Parser;
use pest_derive::Parser; use pest_derive::Parser;
mod preprocessor; mod preprocessor;
pub use preprocessor::*; pub use preprocessor::*;
mod compiler;
pub use compiler::*;
#[derive(Parser)] #[derive(Parser)]
#[grammar = "wgsl.pest"] #[grammar = "wgsl.pest"]
pub(crate) struct WgslParser; pub(crate) struct WgslParser;
fn main() { fn main() {
/* let mut successful_parse = WgslParser::parse(Rule::command_line, "#define_module inner::some_include").unwrap(); tracing_subscriber::fmt()
let p = successful_parse.next().unwrap(); // enable everything
println!("test: {}", p.as_str()); */ .with_max_level(tracing::Level::TRACE)
// sets this to be the default, global collector for this application.
.init();
let mut p = Processor::new(); let mut p = Processor::new();
//let f = p.parse_modules("shaders", ["wgsl"]).unwrap(); //let f = p.parse_modules("shaders", ["wgsl"]).unwrap();
@ -78,7 +74,7 @@ fn main() {
for (module, usages) in &module.import_usages { for (module, usages) in &module.import_usages {
println!(" {}:", module); println!(" {}:", module);
for import in &usages.imports { for import in usages.iter() {
println!( println!(
" {:?} `{}` at {}:", " {:?} `{}` at {}:",
import.ty, import.name, import.start_pos import.ty, import.name, import.start_pos
@ -106,12 +102,6 @@ pub struct ExternalUsage {
start_pos: usize, start_pos: usize,
} }
#[derive(Clone)]
pub struct ImportUsage {
module: String,
imports: Vec<ExternalUsage>,
}
#[derive(Clone)] #[derive(Clone)]
pub struct Import { pub struct Import {
module: String, module: String,
@ -123,7 +113,6 @@ pub struct DefRequirement {
/// None if the requirement is local /// None if the requirement is local
module: Option<String>, module: Option<String>,
name: String, name: String,
ty: ExternalUsageType,
} }
#[derive(Clone)] #[derive(Clone)]
@ -151,7 +140,7 @@ pub struct Module {
item_imports: HashMap<String, Import>, item_imports: HashMap<String, Import>,
/// usages of imported things /// usages of imported things
/// ie `other_module::scalar` /// ie `other_module::scalar`
import_usages: HashMap<String, ImportUsage>, import_usages: HashMap<String, Vec<ExternalUsage>>,
/// Imports of modules /// Imports of modules
/// ///
/// These modules are used along side `import_usages` /// These modules are used along side `import_usages`

277
src/preprocessor.rs
View File

@ -1,10 +1,11 @@
use std::{collections::{HashMap, HashSet, VecDeque}, fmt::Write, fs, path::{Path, PathBuf}}; use std::{collections::{HashMap, HashSet, VecDeque}, fmt::Write, fs, path::Path};
use pest::{iterators::Pair, Parser}; use pest::{iterators::Pair, Parser};
use tracing::{debug, debug_span, instrument};
use crate::{recurse_files, DefRequirement, Definition, ExternalUsageType, Import, Module, Rule, WgslParser}; use crate::{recurse_files, DefRequirement, Definition, Import, Module, Rule, WgslParser};
const RESERVED_WORDS: [&str; 171] = [ const RESERVED_WORDS: [&str; 201] = [
"NULL", "Self", "abstract", "active", "alignas", "alignof", "as", "asm", "asm_fragment", "NULL", "Self", "abstract", "active", "alignas", "alignof", "as", "asm", "asm_fragment",
"async", "attribute", "auto", "await", "become", "binding_array", "cast", "catch", "class", "async", "attribute", "auto", "await", "become", "binding_array", "cast", "catch", "class",
"co_await", "co_return", "co_yield", "coherent", "column_major", "common", "compile", "co_await", "co_return", "co_yield", "coherent", "column_major", "common", "compile",
@ -25,7 +26,24 @@ const RESERVED_WORDS: [&str; 171] = [
"where", "with", "writeonly", "yield", "alias", "break", "case", "const", "const_assert", "where", "with", "writeonly", "yield", "alias", "break", "case", "const", "const_assert",
"continue", "continuing", "default", "diagnostic", "discard", "else", "enable", "false", "fn", "continue", "continuing", "default", "diagnostic", "discard", "else", "enable", "false", "fn",
"for", "if", "let", "loop", "override", "requires", "return", "struct", "switch", "true", "for", "if", "let", "loop", "override", "requires", "return", "struct", "switch", "true",
"var", "while" "var", "while",
// types
"vec4", "vec3", "vec2",
"mat2x2", "mat2x3", "mat2x4",
"mat3x2", "mat3x3", "mat3x4",
"mat4x2", "mat4x3", "mat4x4",
"f16", "f32", "i32", "u32",
"bool", "array", "atomic",
// texture types
"texture_1d",
"texture_2d", "texture_2d_array",
"texture_depth_2d", "texture_depth_2d_array",
"texture_depth_cube", "texture_depth_cube_array",
"texture_3d",
"texture_cube_array",
"sampler", "sampler_comparison",
]; ];
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
@ -54,6 +72,28 @@ impl Processor {
Self::default() Self::default()
} }
fn add_func_requirements(&self, found_requirements: &mut HashSet<String>, requirements: &mut Vec<DefRequirement>, fn_name: &str) {
if found_requirements.contains(fn_name) {
return;
}
found_requirements.insert(fn_name.to_string());
debug!("Found call to `{}`", fn_name);
// ignore reserved words
if RESERVED_WORDS.contains(&fn_name) {
return;
}
let req = DefRequirement {
// module is discovered later
module: None,
name: fn_name.to_string(),
};
requirements.push(req);
}
#[instrument(fields(module = module.name), skip_all)]
fn get_imports_in_block(&mut self, module: &mut Module, block: Pair<Rule>, found_requirements: &mut HashSet<String>) -> Vec<DefRequirement> { fn get_imports_in_block(&mut self, module: &mut Module, block: Pair<Rule>, found_requirements: &mut HashSet<String>) -> Vec<DefRequirement> {
let mut requirements = vec![]; let mut requirements = vec![];
@ -66,48 +106,58 @@ impl Processor {
Rule::shader_code_fn_usage => { Rule::shader_code_fn_usage => {
let mut usage_inner = code.into_inner(); let mut usage_inner = code.into_inner();
let fn_name = usage_inner.next().unwrap().as_str(); let fn_name = usage_inner.next().unwrap().as_str();
let fn_args: Vec<&str> = usage_inner.map(|a| a.as_str()).collect();
if found_requirements.contains(fn_name) { self.add_func_requirements(found_requirements, &mut requirements, fn_name);
continue;
}
found_requirements.insert(fn_name.to_string());
println!("Found call to {} with args: {:?}", fn_name, fn_args);
// ignore reserved words
if RESERVED_WORDS.contains(&fn_name) {
continue;
}
let req = DefRequirement {
// module is discovered later
module: None,
name: fn_name.to_string(),
ty: ExternalUsageType::Function,
};
requirements.push(req);
}, },
Rule::shader_external_fn => { Rule::shader_external_fn => {
let mut pairs = code.into_inner(); let mut usage_inner = code.into_inner();
// shader_external_variable is the only pair for this rule let fn_path = usage_inner.next().unwrap().as_str();
let ident_name = pairs.next().unwrap().as_str().to_string();
println!("external fn: {ident_name}"); if found_requirements.contains(fn_path) {
continue;
}
found_requirements.insert(fn_path.to_string());
let (fn_module, fn_name) = fn_path.rsplit_once("::").unwrap();
debug!("Found call to `{}::{}`", fn_module, fn_name);
let req = DefRequirement {
module: Some(fn_module.into()),
name: fn_name.to_string(),
};
requirements.push(req);
}, },
Rule::shader_external_variable => { Rule::shader_external_variable => {
let pairs = code.into_inner(); let pairs = code.into_inner();
// shader_external_variable is the only pair for this rule // shader_external_variable is the only pair for this rule
let ident_name = pairs.as_str(); let ident_path = pairs.as_str();
println!("external var: {ident_name}"); if found_requirements.contains(ident_path) {
continue;
}
found_requirements.insert(ident_path.to_string());
let (ident_module, ident_name) = ident_path.rsplit_once("::").unwrap();
debug!("Found call to `{}`", ident_name);
// ignore reserved words
if RESERVED_WORDS.contains(&ident_name) {
continue;
}
let req = DefRequirement {
module: Some(ident_module.into()),
name: ident_name.to_string(),
};
requirements.push(req);
}, },
Rule::newline => (), Rule::newline => (),
Rule::cws => (), Rule::cws => (),
Rule::shader_code_char => (), Rule::shader_code_char => (),
Rule::shader_ident => { Rule::shader_ident => {
let ident = code.as_str(); let ident = code.as_str();
println!("Found usage of ident: {}", ident);
if found_requirements.contains(ident) { if found_requirements.contains(ident) {
continue; continue;
@ -123,7 +173,6 @@ impl Processor {
// module is discovered later // module is discovered later
module: None, module: None,
name: ident.to_string(), name: ident.to_string(),
ty: ExternalUsageType::Variable,
}; };
requirements.push(req); requirements.push(req);
}, },
@ -138,14 +187,16 @@ impl Processor {
/// Parse a module file to attempt to find the include identifier. /// Parse a module file to attempt to find the include identifier.
/// ///
/// Returns `None` if the module does not define an include identifier. /// Returns `None` if the module does not define an include identifier.
//#[instrument(skip(self, module_src))]
pub fn parse_module( pub fn parse_module(
&mut self, &mut self,
module_src: &str, module_src: &str,
) -> Result<Option<String>, PreprocessorError> { ) -> Result<Option<String>, PreprocessorError> {
//let unparsed_file = fs::read_to_string(path.as_ref())?; //let current_span = Span::current();
//let e = current_span.entered();
let e = debug_span!("parse_module", module = tracing::field::Empty).entered();
// add a new line to the end of the input to make the grammar happy let module_src = remove_comments(module_src)?;
//let unparsed_file = format!("{unparsed_file}\n");
let file = WgslParser::parse(Rule::file, &module_src)? let file = WgslParser::parse(Rule::file, &module_src)?
.next() .next()
@ -171,7 +222,7 @@ impl Processor {
let types: Vec<String> = inner.map(|t| t.as_str().to_string()).collect(); let types: Vec<String> = inner.map(|t| t.as_str().to_string()).collect();
println!("found import of types from `{}`: `{:?}`", module_name, types); debug!("Found imports from `{}`: `{:?}`", module_name, types);
// add these type imports to imports of the module // add these type imports to imports of the module
module.item_imports.entry(module_name.into()) module.item_imports.entry(module_name.into())
@ -184,16 +235,19 @@ impl Processor {
Rule::import_module_command => { Rule::import_module_command => {
let mut inner = command_line.into_inner(); let mut inner = command_line.into_inner();
let module_name = inner.next().unwrap().as_str(); let module_name = inner.next().unwrap().as_str();
println!("found import of module: {}", module_name); debug!("Found module import: `{}`", module_name);
module.module_imports.insert(module_name.into()); module.module_imports.insert(module_name.into());
}, },
Rule::define_module_command => { Rule::define_module_command => {
let mut shader_file_pairs = command_line.into_inner(); let mut shader_file_pairs = command_line.into_inner();
let shader_file = shader_file_pairs.next().unwrap(); let module_name = shader_file_pairs.next().unwrap();
let shader_file = shader_file.as_str().to_string(); let module_name = module_name.as_str();
module.name = shader_file; e.record("module", module_name);
debug!("Defined module as `{}`", module_name);
module.name = module_name.into();
} }
_ => unreachable!(), _ => unreachable!(),
} }
@ -205,7 +259,7 @@ impl Processor {
let mut pairs = line.clone().into_inner(); let mut pairs = line.clone().into_inner();
// shader_ident is the only pair for this rule // shader_ident is the only pair for this rule
let fn_name = pairs.next().unwrap().as_str().to_string(); let fn_name = pairs.next().unwrap().as_str().to_string();
println!("fn def: {fn_name:?}"); debug!("Found function def: {fn_name}");
let fn_body = pairs.skip(2).next().unwrap(); let fn_body = pairs.skip(2).next().unwrap();
let mut found_reqs = HashSet::default(); let mut found_reqs = HashSet::default();
@ -229,7 +283,7 @@ impl Processor {
let mut pairs = line.clone().into_inner(); let mut pairs = line.clone().into_inner();
// shader_ident is the only pair for this rule // shader_ident is the only pair for this rule
let const_name = pairs.next().unwrap().as_str().to_string(); let const_name = pairs.next().unwrap().as_str().to_string();
println!("const def: {const_name:?}"); debug!("Found const def: `{const_name}`");
let line_span = line.as_span(); let line_span = line.as_span();
let start_pos = line_span.start(); let start_pos = line_span.start();
@ -263,13 +317,10 @@ impl Processor {
}, },
_ => unimplemented!("ran into unhandled rule: {:?}", line.as_span()) _ => unimplemented!("ran into unhandled rule: {:?}", line.as_span())
} }
println!("code: {}", line.as_str());
} }
Rule::cws => (), Rule::cws => (),
Rule::newline => (), Rule::newline => (),
_ => unimplemented!("ran into unhandled rule: {:?}", line.as_span()) _ => unimplemented!("ran into unhandled rule: ({:?}) {:?}", line.as_rule(), line.as_span())
} }
} }
} }
@ -297,6 +348,7 @@ impl Processor {
/// * `path` - The path to search for files in. /// * `path` - The path to search for files in.
/// * `extensions` - The extensions that the discovered files must have. Make sure they have /// * `extensions` - The extensions that the discovered files must have. Make sure they have
/// no leading '.' /// no leading '.'
#[instrument(skip(self, path, extensions))]
pub fn parse_modules<P: AsRef<Path>, const N: usize>( pub fn parse_modules<P: AsRef<Path>, const N: usize>(
&mut self, &mut self,
path: P, path: P,
@ -320,35 +372,42 @@ impl Processor {
Ok(parsed) Ok(parsed)
} }
fn generate_header(&mut self, module_path: &str) -> String { #[instrument(skip(self))]
fn generate_header(&mut self, module_path: &str) -> Result<String, PreprocessorError> {
let module = self.modules.get(module_path).unwrap(); let module = self.modules.get(module_path).unwrap();
let mut output = String::new(); let mut output = String::new();
compile_definitions(&self.modules, module, &mut output); compile_definitions(&self.modules, module, &mut output)?;
Ok(output)
output
} }
#[instrument(skip(self, shader_code_rule, output))]
fn output_shader_code_line(&self, shader_code_rule: Pair<Rule>, output: &mut String) -> Result<(), std::fmt::Error> { fn output_shader_code_line(&self, shader_code_rule: Pair<Rule>, output: &mut String) -> Result<(), std::fmt::Error> {
for line in shader_code_rule.into_inner() { for line in shader_code_rule.into_inner() {
let (pos_line, pos_col) = line.line_col();
match line.as_rule() { match line.as_rule() {
Rule::shader_external_fn | Rule::shader_external_variable => { Rule::shader_external_fn => {
let mut pairs = line.into_inner(); let mut pairs = line.clone().into_inner();
// shader_external_variable is the only pair for this rule let fn_path = pairs.next().unwrap().as_str().to_string();
let ident_name = pairs.next().unwrap().as_str().to_string(); // the rest of the pairs are the arguments for the fn
let fn_args = pairs.as_str();
// remove the module from the identifier and write it to the output // remove the module from the identifier and write it to the output
if let Some((module_name, ident)) = ident_name.rsplit_once("::") { if let Some((_, fn_name)) = fn_path.rsplit_once("::") {
output.write_str(ident)?; output.write_str(fn_name)?;
output.write_fmt(format_args!("({})", fn_args))?;
} else { } else {
// TODO: not really sure how this would get triggered // TODO: not really sure how this would get triggered
unimplemented!( panic!("Unknown error, rule input: {}", line.as_str());
"this function is actually not external, i think"
);
} }
}, },
Rule::shader_external_variable => {
let pairs = line.clone().into_inner();
// the last pair is the name of the variable
let ident_name = pairs.last().unwrap().as_str();
output.write_str(ident_name)?;
},
//shader_external_variable
Rule::shader_fn_def => { Rule::shader_fn_def => {
let mut rule_inner = line.into_inner(); let mut rule_inner = line.into_inner();
@ -381,7 +440,10 @@ impl Processor {
Ok(()) Ok(())
} }
#[instrument(skip(self, module_src))]
pub fn process_file(&mut self, module_path: &str, module_src: &str) -> Result<String, PreprocessorError> { pub fn process_file(&mut self, module_path: &str, module_src: &str) -> Result<String, PreprocessorError> {
let module_src = remove_comments(module_src)?;
let mut out_string = String::new(); let mut out_string = String::new();
// the output will be at least the length of module_src // the output will be at least the length of module_src
out_string.reserve(module_src.len()); out_string.reserve(module_src.len());
@ -390,7 +452,7 @@ impl Processor {
.next() .next()
.unwrap(); // get and unwrap the `file` rule; never fails .unwrap(); // get and unwrap the `file` rule; never fails
let header = self.generate_header(module_path); let header = self.generate_header(module_path)?;
out_string.write_str("// START OF IMPORT HEADER\n")?; out_string.write_str("// START OF IMPORT HEADER\n")?;
out_string.write_str(&header)?; out_string.write_str(&header)?;
out_string.write_str("// END OF IMPORT HEADER\n")?; out_string.write_str("// END OF IMPORT HEADER\n")?;
@ -412,40 +474,6 @@ impl Processor {
Rule::cws => (), Rule::cws => (),
Rule::shader_code_line => { Rule::shader_code_line => {
self.output_shader_code_line(record, &mut out_string)?; self.output_shader_code_line(record, &mut out_string)?;
/* for line in record.into_inner() {
let (pos_line, pos_col) = line.line_col();
match line.as_rule() {
Rule::shader_external_fn | Rule::shader_external_variable => {
let mut pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.next().unwrap().as_str().to_string();
// remove the module from the identifier and write it to the output
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
out_string.write_str(ident)?;
} else {
// TODO: not really sure how this would get triggered
unimplemented!(
"this function is actually not external, i think"
);
}
}
Rule::shader_code | Rule::shader_const_def => {
let input = line.as_str();
out_string.write_str(&input)?;
},
/* Rule::shader_fn_def => {
}, */
Rule::cws => {
let input = line.as_str();
out_string.write_str(&input)?;
},
Rule::newline => (),
_ => unimplemented!("ran into unhandled rule: {:?}", line.as_rule()),
}
} */
}, },
Rule::newline => { Rule::newline => {
let input = record.as_str(); let input = record.as_str();
@ -470,6 +498,7 @@ fn try_find_requirement_module(module: &Module, req_name: &str) -> Option<String
None None
} }
#[instrument(fields(module = module.name), skip_all)]
fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, output: &mut String) -> Result<(), PreprocessorError> { fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, output: &mut String) -> Result<(), PreprocessorError> {
for (_, funcs) in &module.functions { for (_, funcs) in &module.functions {
let mut requirements = VecDeque::from(funcs.requirements.clone()); let mut requirements = VecDeque::from(funcs.requirements.clone());
@ -488,10 +517,10 @@ fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, outpu
.or_else(|| req_module.constants.get(&req.name)) .or_else(|| req_module.constants.get(&req.name))
.unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name)); .unwrap_or_else(|| panic!("invalid import: {} from {}", req.name, module_name));
let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
println!("got req: {}, subreqs: {:?}", req_def.name, sub_req_names);
if !req_def.requirements.is_empty() { if !req_def.requirements.is_empty() {
let sub_req_names: Vec<String> = req_def.requirements.iter().map(|r| r.name.clone()).collect();
debug!("Found requirement: {}, with the following sub-requirements: {:?}", req_def.name, sub_req_names);
let mut requirements_output = String::new(); let mut requirements_output = String::new();
compile_definitions(modules, req_module, &mut requirements_output)?; compile_definitions(modules, req_module, &mut requirements_output)?;
@ -504,9 +533,59 @@ fn compile_definitions(modules: &HashMap<String, Module>, module: &Module, outpu
output.write_fmt(format_args!("// SOURCE {}::{}\n", module_name, req.name))?; output.write_fmt(format_args!("// SOURCE {}::{}\n", module_name, req.name))?;
output.push_str(func_src); output.push_str(func_src);
output.push_str("\n"); output.push_str("\n");
} else {
debug!("Could not find module for `{}`, assuming its local", req.name);
} }
} }
} }
Ok(()) Ok(())
}
#[instrument(skip(text))]
fn remove_comments(text: &str) -> Result<String, std::fmt::Error> {
let mut output = String::new();
output.reserve(text.len());
let mut comment_layers = 0;
for line in text.lines() {
if let Some(line_comment_start) = line.find("//") {
let uncommented = &line[..line_comment_start];
output.write_str(uncommented)?;
continue;
}
let (block_start, block_end) = (line.find("/*"), line.find("*/"));
if let (Some(start), Some(end)) = (block_start, block_end) {
if comment_layers == 0 {
let before = &line[..start];
output.write_str(before)?;
let after = &line[end + 2..];
output.write_str(after)?;
}
} else if let Some(block_comment_start) = block_start {
if comment_layers == 0 {
let uncommented = &line[..block_comment_start];
output.write_str(uncommented)?;
}
comment_layers += 1;
} else if let Some(block_uncomment_start) = block_end {
if comment_layers == 1 {
let uncommented = &line[block_uncomment_start + 2..];
output.write_str(uncommented)?;
}
comment_layers -= 1;
} else if comment_layers == 0 {
output.write_str(line)?;
output.write_str("\n")?;
}
}
Ok(output)
} }

3
src/wgsl.pest
View File

@ -30,7 +30,8 @@ shader_code = { shader_code_block | shader_code_fn_usage | shader_value | shader
// usages of code from another module // usages of code from another module
shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ } shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ }
shader_external_fn = { shader_external_variable ~ "(" ~ ANY* ~ ")" } //shader_fn_args2 = { shader_var_name_type? ~ (ws* ~ "," ~ ws* ~ shader_var_name_type)* }
shader_external_fn = { shader_external_variable ~ "(" ~ shader_code_fn_arg ~ ("," ~ ws* ~ shader_code_fn_arg)* ~ ")" }
shader_external_code = _{ shader_external_fn | shader_external_variable } shader_external_code = _{ shader_external_fn | shader_external_variable }
shader_actual_code_line = _{ shader_external_code | shader_code } shader_actual_code_line = _{ shader_external_code | shader_code }