more extractor methods, extract from files with parallelism

Author: timofei-iatsenko

Cargo.lock

@@ -17,3 +17,6 @@ lto = false
 opt-level = "s"
 # Optimize for performance, this is default so you don't need to specify it
 # opt-level = "z"
+
+[workspace.dependencies]
+swc_core = { version = "56.0.0", default-features = false }

Cargo.toml

@@ -1,5 +1,5 @@
 mod message_extractor;
 mod message_extractor_visitor;
 
-pub use message_extractor::extract_messages;
+pub use message_extractor::{extract_messages, ExtractorOptions};
 pub use message_extractor_visitor::{ExtractedMessage, ExtractionResult};

crates/lingui_extractor/src/lib.rs

@@ -1,10 +1,8 @@
 use crate::message_extractor_visitor::{ExtractionResult, MessageExtractorVisitor};
 use data_encoding::BASE64;
 use lingui_macro::{LinguiMacroFolder, LinguiOptions};
+use serde::{Deserialize, Serialize};
 use std::sync::Arc;
-
-use swc_sourcemap as sourcemap;
-
 use swc_core::common::{Globals, Mark, GLOBALS};
 use swc_core::ecma::transforms::base::resolver;
 use swc_core::{
@@ -15,10 +13,11 @@ use swc_core::{
     },
     ecma::{
         ast::*,
-        parser::{Parser, StringInput, Syntax, TsSyntax},
+        parser::{Parser, StringInput, Syntax},
         visit::VisitWith,
     },
 };
+use swc_sourcemap as sourcemap;
 
 /// Extract inline source map from source code
 /// Looks for sourceMappingURL comments with inline base64 data
@@ -54,22 +53,19 @@ fn extract_inline_sourcemap(source_code: &str) -> Option<sourcemap::SourceMap> {
     None
 }
 
+#[derive(Default, Clone, Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ExtractorOptions {
+    pub parser: Syntax,
+}
+
 /// Extract messages from source code
 pub fn extract_messages(
     source_code: &str,
     filename: &str,
+    options: &ExtractorOptions,
 ) -> Result<ExtractionResult, Box<dyn std::error::Error>> {
-    // Set up parser with JSX support
-    let syntax = Syntax::Typescript(TsSyntax {
-        tsx: true,
-        ..Default::default()
-    });
-
-    // let fm = c.cm.new_source_file(filename.into(), src);
-
     let source_map = Lrc::new(SourceMap::default());
-    // source_map.new_source_file()
-    // let source_file = source_map.new_source_file(FileName::Anon, src.into());
 
     let source_file = source_map.new_source_file(
         Arc::new(FileName::Custom(filename.to_string())),
@@ -78,7 +74,11 @@ pub fn extract_messages(
 
     let comments = Lrc::new(SingleThreadedComments::default());
 
-    let mut parser = Parser::new(syntax, StringInput::from(&*source_file), Some(&comments));
+    let mut parser = Parser::new(
+        options.parser,
+        StringInput::from(&*source_file),
+        Some(&comments),
+    );
 
     let module = parser
         .parse_module()
@@ -111,7 +111,11 @@ pub fn extract_messages(
         let top_level_mark = Mark::new();
 
         program
-            .apply(&mut resolver(unresolved_mark, top_level_mark, false))
+            .apply(&mut resolver(
+                unresolved_mark,
+                top_level_mark,
+                options.parser.typescript(),
+            ))
             .apply(swc_core::ecma::visit::fold_pass(lingui_macro))
             .visit_with(&mut extractor_visitor);
     });

crates/lingui_extractor/src/message_extractor.rs

@@ -1,8 +1,17 @@
-use lingui_extractor::extract_messages;
 use lingui_extractor::ExtractedMessage;
+use lingui_extractor::{extract_messages, ExtractorOptions};
 
 fn extract_and_sort(source_code: &str, filename: &str) -> (Vec<ExtractedMessage>, Vec<String>) {
-    let result = extract_messages(source_code, filename).expect("Failed to extract messages");
+    let options = ExtractorOptions {
+        syntax: Syntax::Typescript(TsSyntax {
+            tsx: true,
+
+            ..Default::default()
+        }),
+    };
+
+    let result =
+        extract_messages(source_code, filename, &options).expect("Failed to extract messages");
     (result.messages, result.warnings)
 }
 
@@ -370,6 +379,7 @@ const msg = /*i18n*/{
 
 use std::fs;
 use std::path::{Path, PathBuf};
+use swc_core::ecma::parser::{Syntax, TsSyntax};
 
 /// Load fixture file from tests/fixtures/
 fn load_fixture(filename: &str) -> String {

crates/lingui_extractor/tests/message_extractor.rs

@@ -13,7 +13,7 @@ serde = "1.0.207"
 serde_json = "1.0.125"
 regex = "1.10.6"
 once_cell = "1.19.0"
-swc_core = { version = "56.0.0", features = [
+swc_core = { workspace = true, features = [
   "ecma_plugin_transform",
   "ecma_utils",
   "ecma_visit",

crates/lingui_macro/Cargo.toml

@@ -1,4 +1,3 @@
-
 [package]
 authors = ["LongYinan <[email protected]>"]
 edition = "2021"
@@ -12,8 +11,11 @@ crate-type = ["cdylib"]
 napi = "3.0.0"
 napi-derive = "3.0.0"
 lingui_extractor = { path = "../crates/lingui_extractor" }
+swc_core = { workspace = true, features = ["base_node", "swc_config"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+anyhow = "1.0.100"
+rayon = "1.10"
 
 [build-dependencies]
 napi-build = "2"