nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
23 kB
710 lines
1use crate::analyse::{ModuleAnalyzerConstructor, TargetSupport};
2use crate::line_numbers::{self, LineNumbers};
3use crate::type_::PRELUDE_MODULE_NAME;
4use crate::{
5 Error, Result, Warning,
6 ast::{SrcSpan, TypedModule, UntypedModule},
7 build::{
8 Mode, Module, Origin, Outcome, Package, SourceFingerprint, Target,
9 elixir_libraries::ElixirLibraries,
10 native_file_copier::NativeFileCopier,
11 package_loader::{CodegenRequired, PackageLoader, StaleTracker},
12 },
13 codegen::{Erlang, ErlangApp, JavaScript, TypeScriptDeclarations},
14 config::PackageConfig,
15 dep_tree, error,
16 io::{BeamCompiler, CommandExecutor, FileSystemReader, FileSystemWriter, Stdio},
17 metadata::ModuleEncoder,
18 parse::extra::ModuleExtra,
19 paths, type_,
20 uid::UniqueIdGenerator,
21 warning::{TypeWarningEmitter, WarningEmitter},
22};
23use askama::Template;
24use ecow::EcoString;
25use std::collections::HashSet;
26use std::{collections::HashMap, fmt::write, time::SystemTime};
27use vec1::Vec1;
28
29use camino::{Utf8Path, Utf8PathBuf};
30
31use super::{ErlangAppCodegenConfiguration, TargetCodegenConfiguration, Telemetry};
32
33pub struct Compiled {
34 pub modules: Vec<Module>,
35 pub module_names: Vec<EcoString>,
36}
37
38#[derive(Debug)]
39pub struct PackageCompiler<'a, IO> {
40 pub io: IO,
41 pub out: &'a Utf8Path,
42 pub lib: &'a Utf8Path,
43 pub root: &'a Utf8Path,
44 pub mode: Mode,
45 pub target: &'a TargetCodegenConfiguration,
46 pub config: &'a PackageConfig,
47 pub ids: UniqueIdGenerator,
48 pub write_metadata: bool,
49 pub perform_codegen: bool,
50 /// If set to false the compiler won't load and analyse any of the package's
51 /// modules and always succeed compilation returning no compile modules.
52 ///
53 /// Code generation is still carried out so that a root package will have an
54 /// entry point nonetheless.
55 ///
56 pub compile_modules: bool,
57 pub write_entrypoint: bool,
58 pub copy_native_files: bool,
59 pub compile_beam_bytecode: bool,
60 pub subprocess_stdio: Stdio,
61 pub target_support: TargetSupport,
62 pub cached_warnings: CachedWarnings,
63}
64
65impl<'a, IO> PackageCompiler<'a, IO>
66where
67 IO: FileSystemReader + FileSystemWriter + CommandExecutor + BeamCompiler + Clone,
68{
69 pub fn new(
70 config: &'a PackageConfig,
71 mode: Mode,
72 root: &'a Utf8Path,
73 out: &'a Utf8Path,
74 lib: &'a Utf8Path,
75 target: &'a TargetCodegenConfiguration,
76 ids: UniqueIdGenerator,
77 io: IO,
78 ) -> Self {
79 Self {
80 io,
81 ids,
82 out,
83 lib,
84 root,
85 mode,
86 config,
87 target,
88 write_metadata: true,
89 perform_codegen: true,
90 compile_modules: true,
91 write_entrypoint: false,
92 copy_native_files: true,
93 compile_beam_bytecode: true,
94 subprocess_stdio: Stdio::Inherit,
95 target_support: TargetSupport::NotEnforced,
96 cached_warnings: CachedWarnings::Ignore,
97 }
98 }
99
100 /// Compile the package.
101 /// Returns a list of modules that were compiled. Any modules that were read
102 /// from the cache will not be returned.
103 // TODO: return the cached modules.
104 pub fn compile(
105 mut self,
106 warnings: &WarningEmitter,
107 existing_modules: &mut im::HashMap<EcoString, type_::ModuleInterface>,
108 already_defined_modules: &mut im::HashMap<EcoString, Utf8PathBuf>,
109 stale_modules: &mut StaleTracker,
110 incomplete_modules: &mut HashSet<EcoString>,
111 telemetry: &dyn Telemetry,
112 ) -> Outcome<Compiled, Error> {
113 let span = tracing::info_span!("compile", package = %self.config.name.as_str());
114 let _enter = span.enter();
115
116 // Ensure that the package is compatible with this version of Gleam
117 if let Err(e) = self.config.check_gleam_compatibility() {
118 return e.into();
119 }
120
121 let artefact_directory = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
122 let codegen_required = if self.perform_codegen {
123 CodegenRequired::Yes
124 } else {
125 CodegenRequired::No
126 };
127
128 let loader = PackageLoader::new(
129 self.io.clone(),
130 self.ids.clone(),
131 self.mode,
132 self.root,
133 self.cached_warnings,
134 warnings,
135 codegen_required,
136 &artefact_directory,
137 self.target.target(),
138 &self.config.name,
139 stale_modules,
140 already_defined_modules,
141 incomplete_modules,
142 );
143
144 let loaded = if self.compile_modules {
145 match loader.run() {
146 Ok(loaded) => loaded,
147 Err(error) => return error.into(),
148 }
149 } else {
150 Loaded::empty()
151 };
152
153 let mut module_names = Vec::new();
154
155 // Load the cached modules that have previously been compiled
156 for module in loaded.cached.into_iter() {
157 // Emit any cached warnings.
158 // Note that `self.cached_warnings` is set to `Ignore` (such as for
159 // dependency packages) then this field will not be populated.
160 if let Err(e) = self.emit_warnings(warnings, &module) {
161 return e.into();
162 }
163
164 module_names.push(module.name.clone());
165
166 // Register the cached module so its type information etc can be
167 // used for compiling futher modules.
168 _ = existing_modules.insert(module.name.clone(), module);
169 }
170
171 if !loaded.to_compile.is_empty() {
172 // Print that work is being done
173 if self.perform_codegen {
174 telemetry.compiling_package(&self.config.name);
175 } else {
176 telemetry.checking_package(&self.config.name)
177 }
178 }
179
180 // Type check the modules that are new or have changed
181 tracing::info!(count=%loaded.to_compile.len(), "analysing_modules");
182 let outcome = analyse(
183 &self.config,
184 self.target.target(),
185 self.mode,
186 &self.ids,
187 loaded.to_compile,
188 existing_modules,
189 warnings,
190 self.target_support,
191 incomplete_modules,
192 );
193
194 let mut modules = match outcome {
195 Outcome::Ok(modules) => modules,
196 Outcome::PartialFailure(modules, error) => {
197 return Outcome::PartialFailure(
198 Compiled {
199 modules,
200 module_names,
201 },
202 error,
203 )
204 }
205 Outcome::TotalFailure(error) => return Outcome::TotalFailure(error),
206 };
207
208 for mut module in modules.iter_mut() {
209 module.attach_doc_and_module_comments();
210 }
211
212 tracing::debug!("performing_code_generation");
213
214 if let Err(error) = self.perform_codegen(&modules) {
215 return error.into();
216 }
217
218 if let Err(error) = self.encode_and_write_metadata(&modules) {
219 return error.into();
220 }
221
222 Outcome::Ok(Compiled {
223 modules,
224 module_names,
225 })
226 }
227
228 fn compile_erlang_to_beam(
229 &mut self,
230 modules: &HashSet<Utf8PathBuf>,
231 ) -> Result<Vec<EcoString>, Error> {
232 if modules.is_empty() {
233 tracing::debug!("no_erlang_to_compile");
234 return Ok(Vec::new());
235 }
236
237 tracing::debug!("compiling_erlang");
238
239 self.io
240 .compile_beam(self.out, self.lib, modules, self.subprocess_stdio)
241 .map(|modules| modules.iter().map(|str| EcoString::from(str)).collect())
242 }
243
244 fn copy_project_native_files(
245 &mut self,
246 destination_dir: &Utf8Path,
247 to_compile_modules: &mut HashSet<Utf8PathBuf>,
248 ) -> Result<(), Error> {
249 tracing::debug!("copying_native_source_files");
250
251 // TODO: unit test
252 let priv_source = self.root.join("priv");
253 let priv_build = self.out.join("priv");
254 if self.io.is_directory(&priv_source) && !self.io.is_directory(&priv_build) {
255 tracing::debug!("linking_priv_to_build");
256 self.io.symlink_dir(&priv_source, &priv_build)?;
257 }
258
259 let copier = NativeFileCopier::new(self.io.clone(), self.root.clone(), destination_dir);
260 let copied = copier.run()?;
261
262 to_compile_modules.extend(copied.to_compile.into_iter());
263
264 // If there are any Elixir files then we need to locate Elixir
265 // installed on this system for use in compilation.
266 if copied.any_elixir {
267 ElixirLibraries::make_available(
268 &self.io,
269 &self.lib.to_path_buf(),
270 self.subprocess_stdio,
271 )?;
272 }
273
274 Ok(())
275 }
276
277 fn encode_and_write_metadata(&mut self, modules: &[Module]) -> Result<()> {
278 if !self.write_metadata {
279 tracing::debug!("package_metadata_writing_disabled");
280 return Ok(());
281 }
282 if modules.is_empty() {
283 return Ok(());
284 }
285
286 let artefact_dir = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
287
288 tracing::debug!("writing_module_caches");
289 for module in modules {
290 let module_name = module.name.replace("/", "@");
291
292 // Write metadata file
293 let name = format!("{}.cache", &module_name);
294 let path = artefact_dir.join(name);
295 let bytes = ModuleEncoder::new(&module.ast.type_info).encode()?;
296 self.io.write_bytes(&path, &bytes)?;
297
298 // Write cache info
299 let name = format!("{}.cache_meta", &module_name);
300 let path = artefact_dir.join(name);
301 let info = CacheMetadata {
302 mtime: module.mtime,
303 codegen_performed: self.perform_codegen,
304 dependencies: module.dependencies.clone(),
305 fingerprint: SourceFingerprint::new(&module.code),
306 line_numbers: module.ast.type_info.line_numbers.clone(),
307 };
308 self.io.write_bytes(&path, &info.to_binary())?;
309
310 // Write warnings.
311 // Dependency packages don't get warnings persisted as the
312 // programmer doesn't want to be told every time about warnings they
313 // cannot fix directly.
314 if self.cached_warnings.should_use() {
315 let name = format!("{}.cache_warnings", &module_name);
316 let path = artefact_dir.join(name);
317 let warnings = &module.ast.type_info.warnings;
318 let data = bincode::serialize(warnings).expect("Serialise warnings");
319 self.io.write_bytes(&path, &data)?;
320 }
321 }
322 Ok(())
323 }
324
325 fn perform_codegen(&mut self, modules: &[Module]) -> Result<()> {
326 if !self.perform_codegen {
327 tracing::debug!("skipping_codegen");
328 return Ok(());
329 }
330
331 match self.target {
332 TargetCodegenConfiguration::JavaScript {
333 emit_typescript_definitions,
334 prelude_location,
335 } => self.perform_javascript_codegen(
336 modules,
337 *emit_typescript_definitions,
338 prelude_location,
339 ),
340 TargetCodegenConfiguration::Erlang { app_file } => {
341 self.perform_erlang_codegen(modules, app_file.as_ref())
342 }
343 }
344 }
345
346 fn perform_erlang_codegen(
347 &mut self,
348 modules: &[Module],
349 app_file_config: Option<&ErlangAppCodegenConfiguration>,
350 ) -> Result<(), Error> {
351 let mut written = HashSet::new();
352 let build_dir = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
353 let include_dir = self.out.join("include");
354 let io = self.io.clone();
355
356 io.mkdir(&build_dir)?;
357
358 if self.copy_native_files {
359 self.copy_project_native_files(&build_dir, &mut written)?;
360 } else {
361 tracing::debug!("skipping_native_file_copying");
362 }
363
364 if self.compile_beam_bytecode && self.write_entrypoint {
365 self.render_erlang_entrypoint_module(&build_dir, &mut written)?;
366 } else {
367 tracing::debug!("skipping_entrypoint_generation");
368 }
369
370 // NOTE: This must come after `copy_project_native_files` to ensure that
371 // we overwrite any precompiled Erlang that was included in the Hex
372 // package. Otherwise we will build the potentially outdated precompiled
373 // version and not the newly compiled version.
374 Erlang::new(&build_dir, &include_dir).render(io.clone(), modules, self.root)?;
375
376 let native_modules: Vec<EcoString> = if self.compile_beam_bytecode {
377 written.extend(modules.iter().map(Module::compiled_erlang_path));
378 self.compile_erlang_to_beam(&written)?
379 } else {
380 tracing::debug!("skipping_erlang_bytecode_compilation");
381 Vec::new()
382 };
383
384 if let Some(config) = app_file_config {
385 ErlangApp::new(&self.out.join("ebin"), config).render(
386 io,
387 &self.config,
388 modules,
389 native_modules,
390 )?;
391 }
392 Ok(())
393 }
394
395 fn perform_javascript_codegen(
396 &mut self,
397 modules: &[Module],
398 typescript: bool,
399 prelude_location: &Utf8Path,
400 ) -> Result<(), Error> {
401 let mut written = HashSet::new();
402 let typescript = if typescript {
403 TypeScriptDeclarations::Emit
404 } else {
405 TypeScriptDeclarations::None
406 };
407
408 JavaScript::new(&self.out, typescript, prelude_location, self.target_support)
409 .render(&self.io, modules)?;
410
411 if self.copy_native_files {
412 self.copy_project_native_files(&self.out, &mut written)?;
413 } else {
414 tracing::debug!("skipping_native_file_copying");
415 }
416
417 Ok(())
418 }
419
420 fn render_erlang_entrypoint_module(
421 &mut self,
422 out: &Utf8Path,
423 modules_to_compile: &mut HashSet<Utf8PathBuf>,
424 ) -> Result<(), Error> {
425 let name = format!("{name}@@main.erl", name = self.config.name);
426 let path = out.join(&name);
427
428 // If the entrypoint module has already been created then we don't need
429 // to write and compile it again.
430 if self.io.is_file(&path) {
431 tracing::debug!("erlang_entrypoint_already_exists");
432 return Ok(());
433 }
434
435 let template = ErlangEntrypointModule {
436 application: &self.config.name,
437 };
438 let module = template.render().expect("Erlang entrypoint rendering");
439 self.io.write(&path, &module)?;
440 let _ = modules_to_compile.insert(name.into());
441 tracing::debug!("erlang_entrypoint_written");
442 Ok(())
443 }
444
445 fn emit_warnings(
446 &self,
447 warnings: &WarningEmitter,
448 module: &type_::ModuleInterface,
449 ) -> Result<()> {
450 for warning in &module.warnings {
451 let src = self.io.read(&module.src_path)?;
452 warnings.emit(Warning::Type {
453 path: module.src_path.clone(),
454 src: src.into(),
455 warning: warning.clone(),
456 });
457 }
458
459 Ok(())
460 }
461}
462
463fn analyse(
464 package_config: &PackageConfig,
465 target: Target,
466 mode: Mode,
467 ids: &UniqueIdGenerator,
468 mut parsed_modules: Vec<UncompiledModule>,
469 module_types: &mut im::HashMap<EcoString, type_::ModuleInterface>,
470 warnings: &WarningEmitter,
471 target_support: TargetSupport,
472 incomplete_modules: &mut HashSet<EcoString>,
473) -> Outcome<Vec<Module>, Error> {
474 let mut modules = Vec::with_capacity(parsed_modules.len() + 1);
475 let direct_dependencies = package_config.dependencies_for(mode).expect("Package deps");
476
477 // Insert the prelude
478 // DUPE: preludeinsertion
479 // TODO: Currently we do this here and also in the tests. It would be better
480 // to have one place where we create all this required state for use in each
481 // place.
482 let _ = module_types.insert(PRELUDE_MODULE_NAME.into(), type_::build_prelude(ids));
483
484 for UncompiledModule {
485 name,
486 code,
487 ast,
488 path,
489 mtime,
490 origin,
491 package,
492 dependencies,
493 extra,
494 } in parsed_modules
495 {
496 tracing::debug!(module = ?name, "Type checking");
497
498 let line_numbers = LineNumbers::new(&code);
499
500 let analysis = crate::analyse::ModuleAnalyzerConstructor {
501 target,
502 ids,
503 origin,
504 importable_modules: module_types,
505 warnings: &TypeWarningEmitter::new(path.clone(), code.clone(), warnings.clone()),
506 direct_dependencies: &direct_dependencies,
507 target_support,
508 package_config,
509 }
510 .infer_module(ast, line_numbers, path.clone());
511
512 match analysis {
513 Outcome::Ok(ast) => {
514 // Module has compiled successfully. Make sure it isn't marked as incomplete.
515 let _ = incomplete_modules.remove(&name.clone());
516 // Register the types from this module so they can be imported into
517 // other modules.
518 let _ = module_types.insert(name.clone(), ast.type_info.clone());
519 // Register the successfully type checked module data so that it can be
520 // used for code generation and in the language server.
521 modules.push(Module {
522 dependencies,
523 origin,
524 extra,
525 mtime,
526 name,
527 code,
528 ast,
529 input_path: path,
530 });
531 }
532
533 Outcome::PartialFailure(ast, errors) => {
534 let error = Error::Type {
535 names: ast.names.clone(),
536 path: path.clone(),
537 src: code.clone(),
538 errors,
539 };
540 // Mark as incomplete so that this module isn't reloaded from cache.
541 let _ = incomplete_modules.insert(name.clone());
542 // Register the partially type checked module data so that it can be
543 // used in the language server.
544 modules.push(Module {
545 dependencies,
546 origin,
547 extra,
548 mtime,
549 name,
550 code,
551 ast,
552 input_path: path,
553 });
554 // WARNING: This cannot be used for code generation as the code has errors.
555 return Outcome::PartialFailure(modules, error);
556 }
557
558 Outcome::TotalFailure(errors) => {
559 return Outcome::TotalFailure(Error::Type {
560 names: Default::default(),
561 path: path.clone(),
562 src: code.clone(),
563 errors,
564 });
565 }
566 };
567 }
568
569 Outcome::Ok(modules)
570}
571
572pub(crate) fn module_name(package_path: &Utf8Path, full_module_path: &Utf8Path) -> EcoString {
573 // /path/to/project/_build/default/lib/the_package/src/my/module.gleam
574
575 // my/module.gleam
576 let mut module_path = full_module_path
577 .strip_prefix(package_path)
578 .expect("Stripping package prefix from module path")
579 .to_path_buf();
580
581 // my/module
582 let _ = module_path.set_extension("");
583
584 // Stringify
585 let name = module_path.to_string();
586
587 // normalise windows paths
588 name.replace("\\", "/").into()
589}
590
591#[derive(Debug)]
592pub(crate) enum Input {
593 New(UncompiledModule),
594 Cached(CachedModule),
595}
596
597impl Input {
598 pub fn name(&self) -> &EcoString {
599 match self {
600 Input::New(m) => &m.name,
601 Input::Cached(m) => &m.name,
602 }
603 }
604
605 pub fn source_path(&self) -> &Utf8Path {
606 match self {
607 Input::New(m) => &m.path,
608 Input::Cached(m) => &m.source_path,
609 }
610 }
611
612 pub fn dependencies(&self) -> Vec<EcoString> {
613 match self {
614 Input::New(m) => m.dependencies.iter().map(|(n, _)| n.clone()).collect(),
615 Input::Cached(m) => m.dependencies.iter().map(|(n, _)| n.clone()).collect(),
616 }
617 }
618
619 /// Returns `true` if the input is [`New`].
620 ///
621 /// [`New`]: Input::New
622 #[must_use]
623 pub(crate) fn is_new(&self) -> bool {
624 matches!(self, Self::New(..))
625 }
626
627 /// Returns `true` if the input is [`Cached`].
628 ///
629 /// [`Cached`]: Input::Cached
630 #[must_use]
631 pub(crate) fn is_cached(&self) -> bool {
632 matches!(self, Self::Cached(..))
633 }
634}
635
636#[derive(Debug)]
637pub(crate) struct CachedModule {
638 pub name: EcoString,
639 pub origin: Origin,
640 pub dependencies: Vec<(EcoString, SrcSpan)>,
641 pub source_path: Utf8PathBuf,
642 pub line_numbers: LineNumbers,
643}
644
645#[derive(Debug, serde::Serialize, serde::Deserialize)]
646pub(crate) struct CacheMetadata {
647 pub mtime: SystemTime,
648 pub codegen_performed: bool,
649 pub dependencies: Vec<(EcoString, SrcSpan)>,
650 pub fingerprint: SourceFingerprint,
651 pub line_numbers: LineNumbers,
652}
653
654impl CacheMetadata {
655 pub fn to_binary(&self) -> Vec<u8> {
656 bincode::serialize(self).expect("Serializing cache info")
657 }
658
659 pub fn from_binary(bytes: &[u8]) -> Result<Self, String> {
660 bincode::deserialize(bytes).map_err(|e| e.to_string())
661 }
662}
663
664#[derive(Debug, Default, PartialEq, Eq)]
665pub(crate) struct Loaded {
666 pub to_compile: Vec<UncompiledModule>,
667 pub cached: Vec<type_::ModuleInterface>,
668}
669
670impl Loaded {
671 fn empty() -> Self {
672 Self {
673 to_compile: vec![],
674 cached: vec![],
675 }
676 }
677}
678
679#[derive(Debug, PartialEq, Eq)]
680pub(crate) struct UncompiledModule {
681 pub path: Utf8PathBuf,
682 pub name: EcoString,
683 pub code: EcoString,
684 pub mtime: SystemTime,
685 pub origin: Origin,
686 pub package: EcoString,
687 pub dependencies: Vec<(EcoString, SrcSpan)>,
688 pub ast: UntypedModule,
689 pub extra: ModuleExtra,
690}
691
692#[derive(Template)]
693#[template(path = "gleam@@main.erl", escape = "none")]
694struct ErlangEntrypointModule<'a> {
695 application: &'a str,
696}
697
698#[derive(Debug, Clone, Copy)]
699pub enum CachedWarnings {
700 Use,
701 Ignore,
702}
703impl CachedWarnings {
704 pub(crate) fn should_use(&self) -> bool {
705 match self {
706 CachedWarnings::Use => true,
707 CachedWarnings::Ignore => false,
708 }
709 }
710}