From 6e4efa973b36e49237d24412f9167f6c4758fce2 Mon Sep 17 00:00:00 2001 From: SeanOMik Date: Fri, 2 Aug 2024 17:17:53 -0400 Subject: [PATCH] parse imports of individual variables and functions as well as modules --- shaders/inner_include.wgsl | 3 +- shaders/simple.wgsl | 5 +- src/main.rs | 291 +++++++++++++++++++++++++------------ src/wgsl.pest | 54 ++++--- 4 files changed, 241 insertions(+), 112 deletions(-) diff --git a/shaders/inner_include.wgsl b/shaders/inner_include.wgsl index 0eed22c..35183d7 100644 --- a/shaders/inner_include.wgsl +++ b/shaders/inner_include.wgsl @@ -3,5 +3,6 @@ const scalar: f32 = 5.0; fn mult_some_nums(a: f32, b: f32) -> f32 { - return a * b; + let c = a * b; + return c; } \ No newline at end of file diff --git a/shaders/simple.wgsl b/shaders/simple.wgsl index 2f92ea2..2314402 100644 --- a/shaders/simple.wgsl +++ b/shaders/simple.wgsl @@ -1,6 +1,7 @@ #define_module simple -#import inner::some_include +#import inner::some_include::{scalar, mult_some_nums} +#import outer fn do_something_cool(in: f32) -> f32 { - return inner::some_include::scalar * mult_some_nums(in, 2.0); + return scalar * mult_some_nums(in, 2.0); } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 1db953b..49e7aa2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,38 +47,48 @@ fn main() { .expect("failed to find module"); for (name, module) in &p.modules { - println!(" {name}:"); + println!("{name}:"); if !module.constants.is_empty() || !module.functions.is_empty() { - println!(" defines:"); - } - - for con in &module.constants { - println!(" const {con}"); + println!(" defines:"); } - for func in &module.functions { - println!(" fn {func}"); + for (name, def) in &module.constants { + println!(" const {name}, {}-{}", def.start_pos, def.end_pos); + } + + for (name, def) in &module.functions { + println!(" fn {name}, {}-{}", def.start_pos, def.end_pos); + } + + println!(" imported modules: {:?}", module.module_imports); + + if !module.type_imports.is_empty() { + println!(" type imports:"); + } + + for (module, usages) in &module.type_imports { + println!(" {}: {:?}", module, usages.imports); } if !module.import_usages.is_empty() { - println!(" usages:"); + println!(" usages:"); } for (module, usages) in &module.import_usages { - println!(" {}:", module); + println!(" {}:", module); for import in &usages.imports { println!( - " {:?} `{}` at L{} C{}:", - import.ty, import.name, import.line, import.col + " {:?} `{}` at {}:", + import.ty, import.name, import.start_pos ); } } } - /* let processed = p.process_file("shaders/base.wgsl").unwrap(); - fs::write("out.wgsl", processed).unwrap(); */ + /* let out = p.process_file("shaders/simple.wgsl").unwrap(); + fs::write("out.wgsl", out).unwrap(); */ } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -90,8 +100,8 @@ pub enum ExternalUsageType { pub struct ExternalUsage { name: String, ty: ExternalUsageType, - line: usize, - col: usize, + /// The start byte position as a `usize`. + start_pos: usize, } pub struct ImportUsage { @@ -99,13 +109,27 @@ pub struct ImportUsage { imports: Vec, } +pub struct Import { + module: String, + imports: Vec, +} + +pub struct Definition { + name: String, + /// The start byte position as a `usize`. + start_pos: usize, + /// The end byte position as a `usize`. + end_pos: usize, +} + #[derive(Default)] pub struct Module { name: String, path: String, - constants: HashSet, - functions: HashSet, - //imports: HashSet, + constants: HashMap, + functions: HashMap, + module_imports: HashSet, + type_imports: HashMap, import_usages: HashMap, } @@ -129,7 +153,7 @@ impl Processor { let unparsed_file = fs::read_to_string(path.as_ref())?; // add a new line to the end of the input to make the grammar happy - let unparsed_file = format!("{unparsed_file}\n"); + //let unparsed_file = format!("{unparsed_file}\n"); let file = WgslParser::parse(Rule::file, &unparsed_file) .map_err(|e| PreprocessorError::ParserError { @@ -150,7 +174,33 @@ impl Processor { let command_line = pairs.next().unwrap(); match command_line.as_rule() { - Rule::import_command => (), + //Rule::import_command => (), + Rule::import_types_command => { + let mut inner = command_line.into_inner(); + let import_module_command = inner.next().unwrap(); + + let mut import_module_command = import_module_command.into_inner(); + let module_name = import_module_command.next().unwrap().as_str(); + + let types: Vec = inner.map(|t| t.as_str().to_string()).collect(); + + println!("found import of types from `{}`: `{:?}`", module_name, types); + + // add these type imports to imports of the module + module.type_imports.entry(module_name.into()) + .or_insert_with(|| Import { + module: module_name.into(), + imports: vec![], + }) + .imports.extend(types.into_iter()); + }, + Rule::import_module_command => { + let mut inner = command_line.into_inner(); + let module_name = inner.next().unwrap().as_str(); + println!("found import of module: {}", module_name); + + module.module_imports.insert(module_name.into()); + }, Rule::define_module_command => { let mut shader_file_pairs = command_line.into_inner(); let shader_file = shader_file_pairs.next().unwrap(); @@ -167,94 +217,73 @@ impl Processor { match line.as_rule() { Rule::shader_fn_def => { - let mut pairs = line.into_inner(); + let mut pairs = line.clone().into_inner(); // shader_ident is the only pair for this rule let fn_name = pairs.next().unwrap().as_str().to_string(); + println!("fn: {fn_name:?}"); - module.functions.insert(fn_name); + let fn_args = pairs.next().unwrap().as_str(); + let ret_type = pairs.next().unwrap().as_str(); + let fn_body_pair = pairs.next().unwrap(); + + let line_span = line.as_span(); + let start_pos = line_span.start(); + let end_pos = line_span.end(); + + module.functions.insert( + fn_name.clone(), + Definition { + name: fn_name, + start_pos, + end_pos, + }, + ); } Rule::shader_const_def => { - let mut pairs = line.into_inner(); + let mut pairs = line.clone().into_inner(); // shader_ident is the only pair for this rule let const_name = pairs.next().unwrap().as_str().to_string(); + println!("const: {const_name:?}"); - module.constants.insert(const_name); + let line_span = line.as_span(); + let start_pos = line_span.start(); + let end_pos = line_span.end(); + + module.constants.insert( + const_name.clone(), + Definition { + name: const_name, + start_pos, + end_pos, + }, + ); } Rule::shader_external_fn => { let mut pairs = line.into_inner(); // shader_external_variable is the only pair for this rule let ident_name = pairs.next().unwrap().as_str().to_string(); - if let Some((module_name, ident)) = ident_name.rsplit_once("::") { - if module.functions.contains(ident) { - // TODO: find a way to avoid conflicting imports. - // maybe this could be done by renaming local variables/functions? - return Err(PreprocessorError::ConflictingImport { - from_module: module_name.into(), - name: ident.into() - }); - } - - let usage = ExternalUsage { - name: ident.into(), - ty: ExternalUsageType::Function, - line: pos_line, - col: pos_col - }; - - module.import_usages.entry(module_name.into()) - .or_insert_with(|| ImportUsage { - module: module_name.into(), - imports: vec![], - }) - .imports.push(usage); - } else { - // TODO: not really sure how this would get triggered - unimplemented!("this function is actually not external, i think"); - } + println!("external fn: {ident_name}"); } Rule::shader_external_variable => { let pairs = line.into_inner(); // shader_external_variable is the only pair for this rule let ident_name = pairs.as_str(); - if let Some((module_name, ident)) = ident_name.rsplit_once("::") { - if module.constants.contains(ident) { - // TODO: find a way to avoid conflicting imports. - // maybe this could be done by renaming local variables/functions? - return Err(PreprocessorError::ConflictingImport { - from_module: module_name.into(), - name: ident.into() - }); - } - - let usage = ExternalUsage { - name: ident.into(), - ty: ExternalUsageType::Variable, - line: pos_line, - col: pos_col - }; - - module.import_usages.entry(module_name.into()) - .or_insert_with(|| ImportUsage { - module: module_name.into(), - imports: vec![], - }) - .imports.push(usage); - } else { - // TODO: not really sure how this would get triggered - unimplemented!("this function is actually not external, i think"); - } - }, - Rule::shader_code => (), /* { - let chunk = line.as_str(); - println!("shader_code: {chunk}"); - }, */ - //Rule::shader_unknown_code => (), - _ => unreachable!(), + println!("external var: {ident_name}"); + } + Rule::shader_code => { + println!("code: {}", line.as_str()); + } + Rule::cws => (), + Rule::newline => (), + _ => { + unimplemented!("ran into unhandled rule: {:?}", line.as_span()); + } } } } + Rule::newline => (), Rule::EOI => (), _ => unreachable!(), } @@ -324,7 +353,7 @@ impl Processor { let command_line = pairs.next().unwrap(); match command_line.as_rule() { - Rule::import_command => { + Rule::import_module_command => { let mut shader_file_pairs = command_line.into_inner(); let shader_file = shader_file_pairs.next().unwrap(); let shader_file = shader_file.as_str(); @@ -337,11 +366,10 @@ impl Processor { } })?; - //let shader_path = parent_dir.join(shader_file); let included_file = self.process_file(imported_mod.path.clone())?; let start_header = - format!("// ==== START OF INCLUDE OF '{}' ====\n", shader_file); + format!("// ==== START OF INCLUDE OF '{}' ====", shader_file); let end_header = format!("\n// ==== END OF INCLUDE OF '{}' ====\n", shader_file); @@ -349,15 +377,94 @@ impl Processor { out_string.write_str(&included_file)?; out_string.write_str(&end_header)?; } + Rule::import_types_command => { + let mut shader_file_pairs = command_line.into_inner(); + let module_path = shader_file_pairs.next().unwrap(); + let module_path = module_path.into_inner().next().unwrap(); + let module_path = module_path.as_str(); + + let imports: Vec<&str> = + shader_file_pairs.map(|i| i.as_str()).collect(); + + println!("found module import: {}", module_path); + println!("imports: {imports:?}"); + todo!(); + /* let imported_mod = self.modules.get(shader_file).ok_or_else(|| { + PreprocessorError::UnknownModule { + from_path: path.as_ref().to_path_buf(), + module: shader_file.into(), + } + })?; */ + } Rule::define_module_command => (), _ => unreachable!(), } } + Rule::cws => (), Rule::shader_code_line => { - let input = record.as_str(); - // add new line to end of input - let input = format!("{input}\n"); + for line in record.into_inner() { + let (pos_line, pos_col) = line.line_col(); + match line.as_rule() { + Rule::shader_external_fn => { + let mut pairs = line.into_inner(); + // shader_external_variable is the only pair for this rule + let ident_name = pairs.next().unwrap().as_str().to_string(); + + if let Some((module_name, ident)) = ident_name.rsplit_once("::") { + /* let usage = ExternalUsage { + name: ident.into(), + ty: ExternalUsageType::Function, + line: pos_line, + col: pos_col + }; */ + + out_string.write_str(ident)?; + } else { + // TODO: not really sure how this would get triggered + unimplemented!( + "this function is actually not external, i think" + ); + } + } + Rule::shader_external_variable => { + let pairs = line.into_inner(); + // shader_external_variable is the only pair for this rule + let ident_name = pairs.as_str(); + + if let Some((module_name, ident)) = ident_name.rsplit_once("::") { + /* let usage = ExternalUsage { + name: ident.into(), + ty: ExternalUsageType::Variable, + line: pos_line, + col: pos_col + }; */ + + out_string.write_str(ident)?; + } else { + // TODO: not really sure how this would get triggered + unimplemented!( + "this function is actually not external, i think" + ); + } + } + /* Rule::shader_fn_def => (), + Rule::shader_const_def => (), + Rule::shader_code => { */ + Rule::shader_code | Rule::shader_const_def | Rule::shader_fn_def => { + let input = line.as_str(); + out_string.write_str(&input)?; + } + Rule::cws => { + let input = line.as_str(); + out_string.write_str(&input)?; + } + _ => unreachable!(), + } + } + } + Rule::newline => { + let input = record.as_str(); out_string.write_str(&input)?; } Rule::EOI => (), diff --git a/src/wgsl.pest b/src/wgsl.pest index dc8a6f2..33ddbf2 100644 --- a/src/wgsl.pest +++ b/src/wgsl.pest @@ -1,46 +1,66 @@ -shader_ident = { (ASCII_ALPHANUMERIC | "_")* } +shader_ident = { (ASCII_ALPHANUMERIC | "_")+ } + +// a shader generic could have multiple generics, i.e., vec4 +//shader_generic_type = { shader_ident ~ "<" ~ shader_ident ~ ">"+ } +//shader_type = { shader_generic_type | shader_ident } +shader_type = { shader_ident ~ ("<" ~ shader_ident ~ ">")? } + shader_module = { shader_ident ~ ( "::" ~ shader_ident)* } -import_command = { "import" ~ ws ~ shader_module } +import_module_command = { "import" ~ ws ~ shader_module } + +import_list = _{ "{" ~ shader_ident ~ (ws* ~ "," ~ ws* ~ shader_ident)* ~ "}" } +import_types_command = { import_module_command ~ "::" ~ import_list } +//import_types_command = { "import" ~ shader_ident ~ ( "::" ~ shader_ident)* ~ "::" ~ import_list } + +import_command = _{ import_types_command | import_module_command } define_module_command = { "define_module" ~ ws ~ shader_module } preproc_prefix = _{ "#" } // a line of preprocessor commands -command_line = { preproc_prefix ~ (define_module_command | import_command) } +command_line = { preproc_prefix ~ (define_module_command | import_command) ~ newline } // all characters used by wgsl -shader_code_char = { "{" | "}" | "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," } -shader_code = { shader_code_char | ASCII_ALPHANUMERIC+ } -// a catch all for some -//shader_unknown_code = { (ws* ~ (shader_code ~ ws*)*) } +shader_code_char = { "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," } +shader_code_block = { "{" ~ newline* ~ (cws* ~ (shader_actual_code_line ~ cws*)* ~ newline)+ ~ cws* ~ "}" } +shader_code = { shader_code_block | shader_code_char | ASCII_ALPHANUMERIC+ } shader_value_num = { ASCII_DIGIT* ~ ( "." ~ ASCII_DIGIT* )? } shader_value_bool = { "true" | "false" } shader_value = { shader_value_bool | shader_value_num } -shader_var_type = { shader_ident ~ ":" ~ ws* ~ shader_ident } -shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" } // defines type of something i.e., `: f32`, `: u32`, etc. +shader_var_type = { ":" ~ ws* ~ shader_type } +shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" } +shader_var_name_type = { shader_ident ~ shader_var_type } +shader_fn_args = { "(" ~ shader_var_name_type ~ (ws* ~ "," ~ ws* ~ shader_var_name_type)* ~ ")" } +// the body of a function, including the opening and closing brackets +shader_fn_body = { "{" ~ newline* ~ (cws* ~ (shader_actual_code_line ~ cws*)* ~ newline)+ ~ "}" } + shader_fn_def = { - "fn" ~ ws ~ shader_ident ~ - "(" ~ shader_var_type ~ (ws* ~ "," ~ ws* ~ shader_var_type)* ~ ")" ~ ws* ~ - ("->" ~ ws* ~ shader_code ~ ws*)? ~ "{"? + "fn" ~ ws ~ shader_ident ~ shader_fn_args ~ ws ~ "->" ~ ws ~ shader_type ~ ws ~ shader_fn_body } -//shader_fn_def = { "fn" ~ ws ~ shader_ident ~ "(a: f32, b: f32) -> f32 {" } // usages of code from another module shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ } shader_external_fn = { shader_external_variable ~ "(" ~ ANY* ~ ")" } shader_external_code = _{ shader_external_fn | shader_external_variable } +shader_actual_code_line = _{ shader_external_code | shader_code } +//shader_actual_code_line = _{ cws* ~ ( (shader_external_code | shader_code) ~ cws*)* } + // a line of shader code, including white space shader_code_line = { shader_fn_def | - shader_const_def | - (ws* ~ ( (shader_external_code | shader_code) ~ ws*)*) + (shader_const_def ~ newline) | + ws* ~ newline } //shader_code_line = { shader_fn_def | shader_const_def | (ws* ~ (shader_external_code | shader_code)* ~ ws*) } -file = { SOI ~ ( (command_line | shader_code_line) ~ NEWLINE)* ~ EOI } +file = { SOI ~ ( (command_line | shader_code_line) )* ~ EOI } // whitespace -ws = _{ " " | "\t" } \ No newline at end of file +ws = _{ " " | "\t" } +// capturing white space +cws = { " " | "\t" } + +newline = { "\n" | "\r\n" | "\r" } \ No newline at end of file