Merge pull request #1 from matrach/language-features

Per language feature flags & some warns

Author: jiegec

Cargo.lock

@@ -5,27 +5,48 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["all_langs"]
+
+cpp = ["dep:clang"]
+javascript = ["dep:boa_parser", "dep:boa_interner"]
+lua = ["dep:full_moon"]
+python = ["dep:rustpython-parser"]
+rust = ["dep:proc-macro2"]
+sql = ["dep:sqlparser"]
+verilog = ["dep:verilog-lang"]
+
+all_langs = ["cpp", "javascript", "lua", "python", "rust", "sql", "verilog"]
+
+
 [dependencies]
 
 anyhow = "1.0.75"
+once_cell = "1.20.2"
 bitvec = "1.0.1"
-clang = "2.0.0"
 regex = "1.10.0"
 rkr-gst = "0.1.2"
 tempfile = "3.8.0"
 walkdir = "2.4.0"
 serde = { version = "1.0.189", features = ["derive"] }
-rustpython-parser = { version = "0.3.0", features = ["full-lexer"] }
-proc-macro2 = { version = "1.0.69", features = ["span-locations"] }
-verilog-lang = { git = "https://github.com/jiegec/verilog-lang" }
 env_logger = "0.10.0"
 log = "0.4.20"
 adler32 = "1.2.0"
 html-escape = "0.2.13"
 clap = { version = "4.4.6", features = ["derive"] }
-sqlparser = "0.38.0"
-boa_parser = "0.17.3"
-boa_interner = "0.17.3"
-boa_ast = "0.17.3"
-full_moon = "0.18.1"
-once_cell = "1.20.2"
+
+# Cpp
+clang = { version = "2.0.0", optional = true }
+# JavaScript
+boa_parser = { version = "0.17.3", optional = true }
+boa_interner = { version = "0.17.3", optional = true }
+# Lua
+full_moon = { version = "0.18.1", optional = true }
+# Python
+rustpython-parser = { version = "0.3.0", features = ["full-lexer"], optional = true }
+# Rust
+proc-macro2 = { version = "1.0.69", features = ["span-locations"], optional = true }
+# SQL
+sqlparser = { version = "0.38.0", optional = true }
+# Verilog
+verilog-lang = { git = "https://github.com/jiegec/verilog-lang", optional = true }

core/Cargo.toml

@@ -97,6 +97,7 @@ fn main() -> anyhow::Result<()> {
         let keys: Vec<&PathBuf> = all_tokens[submission].keys().collect();
 
         if !template_tokens.contains_key(submission) {
+            warn!("No template found for {}!", submission.display());
             continue;
         }
 

core/src/bin/find_pairs.rs

@@ -31,36 +31,43 @@ struct LangInfo {
 
 fn get_lang_info() -> Vec<LangInfo> {
     vec![
+        #[cfg(feature = "cpp")]
         LangInfo {
             name: Language::Cpp,
             extensions: vec!["cpp", "cc", "cxx", "c++", "c", "cu"],
             tokenizer: Box::new(tokenizer::cpp::Cpp),
         },
+        #[cfg(feature = "rust")]
         LangInfo {
             name: Language::Rust,
             extensions: vec!["rs"],
             tokenizer: Box::new(tokenizer::rust::Rust),
         },
+        #[cfg(feature = "verilog")]
         LangInfo {
             name: Language::Verilog,
             extensions: vec!["v"],
             tokenizer: Box::new(tokenizer::verilog::Verilog),
         },
+        #[cfg(feature = "python")]
         LangInfo {
             name: Language::Python,
             extensions: vec!["py"],
             tokenizer: Box::new(tokenizer::python::Python),
         },
+        #[cfg(feature = "sql")]
         LangInfo {
             name: Language::SQL,
             extensions: vec!["sql"],
             tokenizer: Box::new(tokenizer::sql::SQL),
         },
+        #[cfg(feature = "javascript")]
         LangInfo {
             name: Language::JavaScript,
             extensions: vec!["js"],
             tokenizer: Box::new(tokenizer::javascript::JavaScript),
         },
+        #[cfg(feature = "lua")]
         LangInfo {
             name: Language::Lua,
             extensions: vec!["lua"],
@@ -81,7 +88,8 @@ pub fn tokenize(path: &Path) -> anyhow::Result<Vec<Token>> {
             return lang.tokenizer.tokenize(path);
         }
     }
-    Err(anyhow!("Unsupported file extension: {:?}", path))
+    Err(anyhow!("Unsupported file extension: {:?}. \
+    Did you enable a corresponding feature?", path))
 }
 
 pub fn tokenize_str(content: &str, language: Language) -> anyhow::Result<Vec<Token>> {
@@ -90,5 +98,6 @@ pub fn tokenize_str(content: &str, language: Language) -> anyhow::Result<Vec<Tok
             return lang.tokenizer.tokenize_str(content);
         }
     }
-    Err(anyhow!("Unsupported language: {:?}", language))
+    Err(anyhow!("Unsupported language: {:?}. \
+    Did you enable a corresponding feature?", language))
 }

core/src/lang/mod.rs

@@ -1,7 +1,14 @@
+#[cfg(feature="cpp")]
 pub mod cpp;
+#[cfg(feature="javascript")]
 pub mod javascript;
+#[cfg(feature="lua")]
 pub mod lua;
+#[cfg(feature="python")]
 pub mod python;
+#[cfg(feature="rust")]
 pub mod rust;
+#[cfg(feature="sql")]
 pub mod sql;
+#[cfg(feature="verilog")]
 pub mod verilog;