SeanOMik
/
wgsl-preprocessor
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

parse imports of individual variables and functions as well as modules

This commit is contained in:
SeanOMik 2024-08-02 17:17:53 -04:00
parent a3a06e541a
commit 6e4efa973b
4 changed files with 241 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
shaders/inner_include.wgsl
View File

@ -3,5 +3,6 @@
const scalar: f32 = 5.0;
fn mult_some_nums(a: f32, b: f32) -> f32 {
return a * b;
let c = a * b;
return c;
}

5
shaders/simple.wgsl
View File

@ -1,6 +1,7 @@
#define_module simple
#import inner::some_include
#import inner::some_include::{scalar, mult_some_nums}
#import outer
fn do_something_cool(in: f32) -> f32 {
return inner::some_include::scalar * mult_some_nums(in, 2.0);
return scalar * mult_some_nums(in, 2.0);
}

291
src/main.rs
View File

@ -47,38 +47,48 @@ fn main() {
.expect("failed to find module");
for (name, module) in &p.modules {
println!(" {name}:");
println!("{name}:");
if !module.constants.is_empty() || !module.functions.is_empty() {
println!(" defines:");
}
for con in &module.constants {
println!(" const {con}");
println!(" defines:");
}
for func in &module.functions {
println!(" fn {func}");
for (name, def) in &module.constants {
println!(" const {name}, {}-{}", def.start_pos, def.end_pos);
}
for (name, def) in &module.functions {
println!(" fn {name}, {}-{}", def.start_pos, def.end_pos);
}
println!(" imported modules: {:?}", module.module_imports);
if !module.type_imports.is_empty() {
println!(" type imports:");
}
for (module, usages) in &module.type_imports {
println!(" {}: {:?}", module, usages.imports);
}
if !module.import_usages.is_empty() {
println!(" usages:");
println!(" usages:");
}
for (module, usages) in &module.import_usages {
println!(" {}:", module);
println!(" {}:", module);
for import in &usages.imports {
println!(
" {:?} `{}` at L{} C{}:",
import.ty, import.name, import.line, import.col
" {:?} `{}` at {}:",
import.ty, import.name, import.start_pos
);
}
}
}
/* let processed = p.process_file("shaders/base.wgsl").unwrap();
fs::write("out.wgsl", processed).unwrap(); */
/* let out = p.process_file("shaders/simple.wgsl").unwrap();
fs::write("out.wgsl", out).unwrap(); */
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
@ -90,8 +100,8 @@ pub enum ExternalUsageType {
pub struct ExternalUsage {
name: String,
ty: ExternalUsageType,
line: usize,
col: usize,
/// The start byte position as a `usize`.
start_pos: usize,
}
pub struct ImportUsage {
@ -99,13 +109,27 @@ pub struct ImportUsage {
imports: Vec<ExternalUsage>,
}
pub struct Import {
module: String,
imports: Vec<String>,
}
pub struct Definition {
name: String,
/// The start byte position as a `usize`.
start_pos: usize,
/// The end byte position as a `usize`.
end_pos: usize,
}
#[derive(Default)]
pub struct Module {
name: String,
path: String,
constants: HashSet<String>,
functions: HashSet<String>,
//imports: HashSet<String>,
constants: HashMap<String, Definition>,
functions: HashMap<String, Definition>,
module_imports: HashSet<String>,
type_imports: HashMap<String, Import>,
import_usages: HashMap<String, ImportUsage>,
}
@ -129,7 +153,7 @@ impl Processor {
let unparsed_file = fs::read_to_string(path.as_ref())?;
// add a new line to the end of the input to make the grammar happy
let unparsed_file = format!("{unparsed_file}\n");
//let unparsed_file = format!("{unparsed_file}\n");
let file = WgslParser::parse(Rule::file, &unparsed_file)
.map_err(|e| PreprocessorError::ParserError {
@ -150,7 +174,33 @@ impl Processor {
let command_line = pairs.next().unwrap();
match command_line.as_rule() {
Rule::import_command => (),
//Rule::import_command => (),
Rule::import_types_command => {
let mut inner = command_line.into_inner();
let import_module_command = inner.next().unwrap();
let mut import_module_command = import_module_command.into_inner();
let module_name = import_module_command.next().unwrap().as_str();
let types: Vec<String> = inner.map(|t| t.as_str().to_string()).collect();
println!("found import of types from `{}`: `{:?}`", module_name, types);
// add these type imports to imports of the module
module.type_imports.entry(module_name.into())
.or_insert_with(|| Import {
module: module_name.into(),
imports: vec![],
})
.imports.extend(types.into_iter());
},
Rule::import_module_command => {
let mut inner = command_line.into_inner();
let module_name = inner.next().unwrap().as_str();
println!("found import of module: {}", module_name);
module.module_imports.insert(module_name.into());
},
Rule::define_module_command => {
let mut shader_file_pairs = command_line.into_inner();
let shader_file = shader_file_pairs.next().unwrap();
@ -167,94 +217,73 @@ impl Processor {
match line.as_rule() {
Rule::shader_fn_def => {
let mut pairs = line.into_inner();
let mut pairs = line.clone().into_inner();
// shader_ident is the only pair for this rule
let fn_name = pairs.next().unwrap().as_str().to_string();
println!("fn: {fn_name:?}");
module.functions.insert(fn_name);
let fn_args = pairs.next().unwrap().as_str();
let ret_type = pairs.next().unwrap().as_str();
let fn_body_pair = pairs.next().unwrap();
let line_span = line.as_span();
let start_pos = line_span.start();
let end_pos = line_span.end();
module.functions.insert(
fn_name.clone(),
Definition {
name: fn_name,
start_pos,
end_pos,
},
);
}
Rule::shader_const_def => {
let mut pairs = line.into_inner();
let mut pairs = line.clone().into_inner();
// shader_ident is the only pair for this rule
let const_name = pairs.next().unwrap().as_str().to_string();
println!("const: {const_name:?}");
module.constants.insert(const_name);
let line_span = line.as_span();
let start_pos = line_span.start();
let end_pos = line_span.end();
module.constants.insert(
const_name.clone(),
Definition {
name: const_name,
start_pos,
end_pos,
},
);
}
Rule::shader_external_fn => {
let mut pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.next().unwrap().as_str().to_string();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
if module.functions.contains(ident) {
// TODO: find a way to avoid conflicting imports.
// maybe this could be done by renaming local variables/functions?
return Err(PreprocessorError::ConflictingImport {
from_module: module_name.into(),
name: ident.into()
});
}
let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Function,
line: pos_line,
col: pos_col
};
module.import_usages.entry(module_name.into())
.or_insert_with(|| ImportUsage {
module: module_name.into(),
imports: vec![],
})
.imports.push(usage);
} else {
// TODO: not really sure how this would get triggered
unimplemented!("this function is actually not external, i think");
}
println!("external fn: {ident_name}");
}
Rule::shader_external_variable => {
let pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.as_str();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
if module.constants.contains(ident) {
// TODO: find a way to avoid conflicting imports.
// maybe this could be done by renaming local variables/functions?
return Err(PreprocessorError::ConflictingImport {
from_module: module_name.into(),
name: ident.into()
});
}
let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Variable,
line: pos_line,
col: pos_col
};
module.import_usages.entry(module_name.into())
.or_insert_with(|| ImportUsage {
module: module_name.into(),
imports: vec![],
})
.imports.push(usage);
} else {
// TODO: not really sure how this would get triggered
unimplemented!("this function is actually not external, i think");
}
},
Rule::shader_code => (), /* {
let chunk = line.as_str();
println!("shader_code: {chunk}");
}, */
//Rule::shader_unknown_code => (),
_ => unreachable!(),
println!("external var: {ident_name}");
}
Rule::shader_code => {
println!("code: {}", line.as_str());
}
Rule::cws => (),
Rule::newline => (),
_ => {
unimplemented!("ran into unhandled rule: {:?}", line.as_span());
}
}
}
}
Rule::newline => (),
Rule::EOI => (),
_ => unreachable!(),
}
@ -324,7 +353,7 @@ impl Processor {
let command_line = pairs.next().unwrap();
match command_line.as_rule() {
Rule::import_command => {
Rule::import_module_command => {
let mut shader_file_pairs = command_line.into_inner();
let shader_file = shader_file_pairs.next().unwrap();
let shader_file = shader_file.as_str();
@ -337,11 +366,10 @@ impl Processor {
}
})?;
//let shader_path = parent_dir.join(shader_file);
let included_file = self.process_file(imported_mod.path.clone())?;
let start_header =
format!("// ==== START OF INCLUDE OF '{}' ====\n", shader_file);
format!("// ==== START OF INCLUDE OF '{}' ====", shader_file);
let end_header =
format!("\n// ==== END OF INCLUDE OF '{}' ====\n", shader_file);
@ -349,15 +377,94 @@ impl Processor {
out_string.write_str(&included_file)?;
out_string.write_str(&end_header)?;
}
Rule::import_types_command => {
let mut shader_file_pairs = command_line.into_inner();
let module_path = shader_file_pairs.next().unwrap();
let module_path = module_path.into_inner().next().unwrap();
let module_path = module_path.as_str();
let imports: Vec<&str> =
shader_file_pairs.map(|i| i.as_str()).collect();
println!("found module import: {}", module_path);
println!("imports: {imports:?}");
todo!();
/* let imported_mod = self.modules.get(shader_file).ok_or_else(|| {
PreprocessorError::UnknownModule {
from_path: path.as_ref().to_path_buf(),
module: shader_file.into(),
}
})?; */
}
Rule::define_module_command => (),
_ => unreachable!(),
}
}
Rule::cws => (),
Rule::shader_code_line => {
let input = record.as_str();
// add new line to end of input
let input = format!("{input}\n");
for line in record.into_inner() {
let (pos_line, pos_col) = line.line_col();
match line.as_rule() {
Rule::shader_external_fn => {
let mut pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.next().unwrap().as_str().to_string();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
/* let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Function,
line: pos_line,
col: pos_col
}; */
out_string.write_str(ident)?;
} else {
// TODO: not really sure how this would get triggered
unimplemented!(
"this function is actually not external, i think"
);
}
}
Rule::shader_external_variable => {
let pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.as_str();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
/* let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Variable,
line: pos_line,
col: pos_col
}; */
out_string.write_str(ident)?;
} else {
// TODO: not really sure how this would get triggered
unimplemented!(
"this function is actually not external, i think"
);
}
}
/* Rule::shader_fn_def => (),
Rule::shader_const_def => (),
Rule::shader_code => { */
Rule::shader_code | Rule::shader_const_def | Rule::shader_fn_def => {
let input = line.as_str();
out_string.write_str(&input)?;
}
Rule::cws => {
let input = line.as_str();
out_string.write_str(&input)?;
}
_ => unreachable!(),
}
}
}
Rule::newline => {
let input = record.as_str();
out_string.write_str(&input)?;
}
Rule::EOI => (),

54
src/wgsl.pest
View File

@ -1,46 +1,66 @@
shader_ident = { (ASCII_ALPHANUMERIC | "_")* }
shader_ident = { (ASCII_ALPHANUMERIC | "_")+ }
// a shader generic could have multiple generics, i.e., vec4<f32>
//shader_generic_type = { shader_ident ~ "<" ~ shader_ident ~ ">"+ }
//shader_type = { shader_generic_type | shader_ident }
shader_type = { shader_ident ~ ("<" ~ shader_ident ~ ">")? }
shader_module = { shader_ident ~ ( "::" ~ shader_ident)* }
import_command = { "import" ~ ws ~ shader_module }
import_module_command = { "import" ~ ws ~ shader_module }
import_list = _{ "{" ~ shader_ident ~ (ws* ~ "," ~ ws* ~ shader_ident)* ~ "}" }
import_types_command = { import_module_command ~ "::" ~ import_list }
//import_types_command = { "import" ~ shader_ident ~ ( "::" ~ shader_ident)* ~ "::" ~ import_list }
import_command = _{ import_types_command | import_module_command }
define_module_command = { "define_module" ~ ws ~ shader_module }
preproc_prefix = _{ "#" }
// a line of preprocessor commands
command_line = { preproc_prefix ~ (define_module_command | import_command) }
command_line = { preproc_prefix ~ (define_module_command | import_command) ~ newline }
// all characters used by wgsl
shader_code_char = { "{" | "}" | "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," }
shader_code = { shader_code_char | ASCII_ALPHANUMERIC+ }
// a catch all for some
//shader_unknown_code = { (ws* ~ (shader_code ~ ws*)*) }
shader_code_char = { "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," }
shader_code_block = { "{" ~ newline* ~ (cws* ~ (shader_actual_code_line ~ cws*)* ~ newline)+ ~ cws* ~ "}" }
shader_code = { shader_code_block | shader_code_char | ASCII_ALPHANUMERIC+ }
shader_value_num = { ASCII_DIGIT* ~ ( "." ~ ASCII_DIGIT* )? }
shader_value_bool = { "true" | "false" }
shader_value = { shader_value_bool | shader_value_num }
shader_var_type = { shader_ident ~ ":" ~ ws* ~ shader_ident }
shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" }
// defines type of something i.e., `: f32`, `: u32`, etc.
shader_var_type = { ":" ~ ws* ~ shader_type }
shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" }
shader_var_name_type = { shader_ident ~ shader_var_type }
shader_fn_args = { "(" ~ shader_var_name_type ~ (ws* ~ "," ~ ws* ~ shader_var_name_type)* ~ ")" }
// the body of a function, including the opening and closing brackets
shader_fn_body = { "{" ~ newline* ~ (cws* ~ (shader_actual_code_line ~ cws*)* ~ newline)+ ~ "}" }
shader_fn_def = {
"fn" ~ ws ~ shader_ident ~
"(" ~ shader_var_type ~ (ws* ~ "," ~ ws* ~ shader_var_type)* ~ ")" ~ ws* ~
("->" ~ ws* ~ shader_code ~ ws*)? ~ "{"?
"fn" ~ ws ~ shader_ident ~ shader_fn_args ~ ws ~ "->" ~ ws ~ shader_type ~ ws ~ shader_fn_body
}
//shader_fn_def = { "fn" ~ ws ~ shader_ident ~ "(a: f32, b: f32) -> f32 {" }
// usages of code from another module
shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ }
shader_external_fn = { shader_external_variable ~ "(" ~ ANY* ~ ")" }
shader_external_code = _{ shader_external_fn | shader_external_variable }
shader_actual_code_line = _{ shader_external_code | shader_code }
//shader_actual_code_line = _{ cws* ~ ( (shader_external_code | shader_code) ~ cws*)* }
// a line of shader code, including white space
shader_code_line = {
shader_fn_def |
shader_const_def |
(ws* ~ ( (shader_external_code | shader_code) ~ ws*)*)
(shader_const_def ~ newline) |
ws* ~ newline
}
//shader_code_line = { shader_fn_def | shader_const_def | (ws* ~ (shader_external_code | shader_code)* ~ ws*) }
file = { SOI ~ ( (command_line | shader_code_line) ~ NEWLINE)* ~ EOI }
file = { SOI ~ ( (command_line | shader_code_line) )* ~ EOI }
// whitespace
ws = _{ " " | "\t" }
ws = _{ " " | "\t" }
// capturing white space
cws = { " " | "\t" }
newline = { "\n" | "\r\n" | "\r" }