SeanOMik
/
wgsl-preprocessor
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

parse external usages of variables and functions

This commit is contained in:
SeanOMik 2024-07-31 20:59:09 -04:00
parent d13c7ae129
commit a3a06e541a
7 changed files with 254 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug shader_prepoc",
"cargo": {
"args": [
"build",
//"--manifest-path", "${workspaceFolder}/examples/testbed/Cargo.toml"
"--bin=shader_prepoc",
],
"filter": {
"name": "shader_prepoc",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}"
}
]
}

16
Cargo.lock generated
View File

@ -46,6 +46,12 @@ dependencies = [
"crypto-common",
]
[[package]]
name = "either"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
name = "generic-array"
version = "0.14.7"
@ -56,6 +62,15 @@ dependencies = [
"version_check",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "libc"
version = "0.2.155"
@ -152,6 +167,7 @@ dependencies = [
name = "shader_prepoc"
version = "0.1.0"
dependencies = [
"itertools",
"pest",
"pest_derive",
"thiserror",

1
Cargo.toml
View File

@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
itertools = "0.13.0"
pest = "2.7.11"
pest_derive = "2.7.11"
thiserror = "1.0.63"

2
shaders/inner_include.wgsl
View File

@ -1,5 +1,7 @@
#define_module inner::some_include
const scalar: f32 = 5.0;
fn mult_some_nums(a: f32, b: f32) -> f32 {
return a * b;
}

2
shaders/simple.wgsl
View File

@ -2,5 +2,5 @@
#import inner::some_include
fn do_something_cool(in: f32) -> f32 {
return mult_some_nums(in, 2.0);
return inner::some_include::scalar * mult_some_nums(in, 2.0);
}

196
src/main.rs
View File

@ -1,7 +1,11 @@
use std::{
collections::HashMap, fmt::Write, fs, path::{Path, PathBuf}
collections::{HashMap, HashSet},
fmt::Write,
fs,
path::{Path, PathBuf},
};
use itertools::Itertools;
use pest::Parser;
use pest_derive::Parser;
@ -22,6 +26,8 @@ pub enum PreprocessorError {
FormatError(#[from] std::fmt::Error),
#[error("unknown module import '{module}', in {from_path}")]
UnknownModule { from_path: PathBuf, module: String },
#[error("import usage from `{from_module}` conflicts with local variable/function: `{name}`")]
ConflictingImport { from_module: String, name: String },
}
fn main() {
@ -30,21 +36,77 @@ fn main() {
println!("test: {}", p.as_str()); */
let mut p = Processor::new();
let f = p.parse_modules("shaders", ["wgsl"]).unwrap();
println!("Parsed {} modules:", f);
//let f = p.parse_modules("shaders", ["wgsl"]).unwrap();
//println!("Parsed {} modules:", f);
p.parse_module("shaders/inner_include.wgsl")
.unwrap()
.expect("failed to find module");
for mod_name in p.modules.keys() {
println!(" {mod_name}");
p.parse_module("shaders/simple.wgsl")
.unwrap()
.expect("failed to find module");
for (name, module) in &p.modules {
println!(" {name}:");
if !module.constants.is_empty() || !module.functions.is_empty() {
println!(" defines:");
}
let processed = p.process_file("shaders/base.wgsl").unwrap();
fs::write("out.wgsl", processed).unwrap();
for con in &module.constants {
println!(" const {con}");
}
for func in &module.functions {
println!(" fn {func}");
}
if !module.import_usages.is_empty() {
println!(" usages:");
}
for (module, usages) in &module.import_usages {
println!(" {}:", module);
for import in &usages.imports {
println!(
" {:?} `{}` at L{} C{}:",
import.ty, import.name, import.line, import.col
);
}
}
}
/* let processed = p.process_file("shaders/base.wgsl").unwrap();
fs::write("out.wgsl", processed).unwrap(); */
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ExternalUsageType {
Variable,
Function,
}
pub struct ExternalUsage {
name: String,
ty: ExternalUsageType,
line: usize,
col: usize,
}
pub struct ImportUsage {
module: String,
imports: Vec<ExternalUsage>,
}
#[derive(Default)]
pub struct Module {
name: String,
path: String,
constants: HashSet<String>,
functions: HashSet<String>,
//imports: HashSet<String>,
import_usages: HashMap<String, ImportUsage>,
}
#[derive(Default)]
@ -77,6 +139,9 @@ impl Processor {
.next()
.unwrap(); // get and unwrap the `file` rule; never fails
let mut module = Module::default();
module.path = path.as_ref().to_str().unwrap().into();
for record in file.into_inner() {
match record.as_rule() {
Rule::command_line => {
@ -91,26 +156,118 @@ impl Processor {
let shader_file = shader_file_pairs.next().unwrap();
let shader_file = shader_file.as_str().to_string();
self.modules.insert(
shader_file.clone(),
Module {
name: shader_file.clone(),
path: path.as_ref().to_str().unwrap().into(),
},
);
return Ok(Some(shader_file.clone()));
module.name = shader_file;
}
_ => unreachable!(),
}
}
Rule::shader_code_line => (),
Rule::shader_code_line => {
for line in record.into_inner() {
let (pos_line, pos_col) = line.line_col();
match line.as_rule() {
Rule::shader_fn_def => {
let mut pairs = line.into_inner();
// shader_ident is the only pair for this rule
let fn_name = pairs.next().unwrap().as_str().to_string();
module.functions.insert(fn_name);
}
Rule::shader_const_def => {
let mut pairs = line.into_inner();
// shader_ident is the only pair for this rule
let const_name = pairs.next().unwrap().as_str().to_string();
module.constants.insert(const_name);
}
Rule::shader_external_fn => {
let mut pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.next().unwrap().as_str().to_string();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
if module.functions.contains(ident) {
// TODO: find a way to avoid conflicting imports.
// maybe this could be done by renaming local variables/functions?
return Err(PreprocessorError::ConflictingImport {
from_module: module_name.into(),
name: ident.into()
});
}
let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Function,
line: pos_line,
col: pos_col
};
module.import_usages.entry(module_name.into())
.or_insert_with(|| ImportUsage {
module: module_name.into(),
imports: vec![],
})
.imports.push(usage);
} else {
// TODO: not really sure how this would get triggered
unimplemented!("this function is actually not external, i think");
}
}
Rule::shader_external_variable => {
let pairs = line.into_inner();
// shader_external_variable is the only pair for this rule
let ident_name = pairs.as_str();
if let Some((module_name, ident)) = ident_name.rsplit_once("::") {
if module.constants.contains(ident) {
// TODO: find a way to avoid conflicting imports.
// maybe this could be done by renaming local variables/functions?
return Err(PreprocessorError::ConflictingImport {
from_module: module_name.into(),
name: ident.into()
});
}
let usage = ExternalUsage {
name: ident.into(),
ty: ExternalUsageType::Variable,
line: pos_line,
col: pos_col
};
module.import_usages.entry(module_name.into())
.or_insert_with(|| ImportUsage {
module: module_name.into(),
imports: vec![],
})
.imports.push(usage);
} else {
// TODO: not really sure how this would get triggered
unimplemented!("this function is actually not external, i think");
}
},
Rule::shader_code => (), /* {
let chunk = line.as_str();
println!("shader_code: {chunk}");
}, */
//Rule::shader_unknown_code => (),
_ => unreachable!(),
}
}
}
Rule::EOI => (),
_ => unreachable!(),
}
}
if module.name.is_empty() {
Ok(None)
} else {
let name = module.name.clone();
self.modules.insert(name.clone(), module);
Ok(Some(name))
}
}
/// Find files recursively in `path` with an extension in `extensions`, and parse them.
@ -164,7 +321,6 @@ impl Processor {
Rule::command_line => {
// the parser has found a preprocessor command, figure out what it is
let mut pairs = record.into_inner();
//assert_eq!(pairs.next().unwrap().as_rule(), Rule::preproc_prefix); // skip preproc_prefix
let command_line = pairs.next().unwrap();
match command_line.as_rule() {
@ -172,8 +328,6 @@ impl Processor {
let mut shader_file_pairs = command_line.into_inner();
let shader_file = shader_file_pairs.next().unwrap();
let shader_file = shader_file.as_str();
// remove surrounding quotes
//let shader_file = &shader_file[1..shader_file.len() - 1];
println!("found module import: {}", shader_file);
let imported_mod = self.modules.get(shader_file).ok_or_else(|| {
@ -194,7 +348,7 @@ impl Processor {
out_string.write_str(&start_header)?;
out_string.write_str(&included_file)?;
out_string.write_str(&end_header)?;
},
}
Rule::define_module_command => (),
_ => unreachable!(),
}

36
src/wgsl.pest
View File

@ -1,5 +1,5 @@
shader_module_text = { ASCII_ALPHANUMERIC | "_" | "-" }
shader_module = { shader_module_text* ~ ( "::" ~ shader_module_text*)* }
shader_ident = { (ASCII_ALPHANUMERIC | "_")* }
shader_module = { shader_ident ~ ( "::" ~ shader_ident)* }
import_command = { "import" ~ ws ~ shader_module }
define_module_command = { "define_module" ~ ws ~ shader_module }
preproc_prefix = _{ "#" }
@ -8,9 +8,37 @@ preproc_prefix = _{ "#" }
command_line = { preproc_prefix ~ (define_module_command | import_command) }
// all characters used by wgsl
shader_code = { ASCII_ALPHANUMERIC | "{" | "}" | "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," }
shader_code_char = { "{" | "}" | "@" | "-" | "+" | "*" | "/" | "=" | "(" | ")" | ">" | "<" | ";" | ":" | "." | "_" | "," }
shader_code = { shader_code_char | ASCII_ALPHANUMERIC+ }
// a catch all for some
//shader_unknown_code = { (ws* ~ (shader_code ~ ws*)*) }
shader_value_num = { ASCII_DIGIT* ~ ( "." ~ ASCII_DIGIT* )? }
shader_value_bool = { "true" | "false" }
shader_value = { shader_value_bool | shader_value_num }
shader_var_type = { shader_ident ~ ":" ~ ws* ~ shader_ident }
shader_const_def = { "const" ~ ws ~ shader_ident ~ (ws* ~ shader_var_type)? ~ ws* ~ "=" ~ ws* ~ shader_value ~ ";" }
// defines type of something i.e., `: f32`, `: u32`, etc.
shader_fn_def = {
"fn" ~ ws ~ shader_ident ~
"(" ~ shader_var_type ~ (ws* ~ "," ~ ws* ~ shader_var_type)* ~ ")" ~ ws* ~
("->" ~ ws* ~ shader_code ~ ws*)? ~ "{"?
}
//shader_fn_def = { "fn" ~ ws ~ shader_ident ~ "(a: f32, b: f32) -> f32 {" }
// usages of code from another module
shader_external_variable = { shader_ident ~ ( "::" ~ shader_ident)+ }
shader_external_fn = { shader_external_variable ~ "(" ~ ANY* ~ ")" }
shader_external_code = _{ shader_external_fn | shader_external_variable }
// a line of shader code, including white space
shader_code_line = { (ws* ~ shader_code ~ ws*)* }
shader_code_line = {
shader_fn_def |
shader_const_def |
(ws* ~ ( (shader_external_code | shader_code) ~ ws*)*)
}
//shader_code_line = { shader_fn_def | shader_const_def | (ws* ~ (shader_external_code | shader_code)* ~ ws*) }
file = { SOI ~ ( (command_line | shader_code_line) ~ NEWLINE)* ~ EOI }