nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
25 kB
784 lines
1#[cfg(test)]
2mod tests;
3
4use crate::analyse::{ModuleAnalyzerConstructor, TargetSupport};
5use crate::build::package_loader::CacheFiles;
6
7use crate::error::DefinedModuleOrigin;
8use crate::io::files_with_extension;
9use crate::line_numbers::{self, LineNumbers};
10use crate::type_::PRELUDE_MODULE_NAME;
11use crate::{
12 Error, Result, Warning,
13 ast::{SrcSpan, TypedModule, UntypedModule},
14 build::{
15 Mode, Module, Origin, Outcome, Package, SourceFingerprint, Target,
16 elixir_libraries::ElixirLibraries,
17 native_file_copier::NativeFileCopier,
18 package_loader::{CodegenRequired, PackageLoader, StaleTracker},
19 },
20 codegen::{Erlang, ErlangApp, JavaScript, TypeScriptDeclarations},
21 config::PackageConfig,
22 dep_tree, error,
23 io::{BeamCompiler, CommandExecutor, FileSystemReader, FileSystemWriter, Stdio},
24 parse::extra::ModuleExtra,
25 paths, type_,
26 uid::UniqueIdGenerator,
27 warning::{TypeWarningEmitter, WarningEmitter},
28};
29use crate::{inline, metadata};
30use askama::Template;
31use ecow::EcoString;
32use std::collections::HashSet;
33use std::{collections::HashMap, fmt::write, time::SystemTime};
34use vec1::Vec1;
35
36use camino::{Utf8Path, Utf8PathBuf};
37
38use super::{ErlangAppCodegenConfiguration, TargetCodegenConfiguration, Telemetry};
39
40#[derive(Debug)]
41pub struct Compiled {
42 /// The modules which were just compiled
43 pub modules: Vec<Module>,
44 /// The names of all cached modules, which are not present in the `modules` field.
45 pub cached_module_names: Vec<EcoString>,
46}
47
48#[derive(Debug)]
49pub struct PackageCompiler<'a, IO> {
50 pub io: IO,
51 pub out: &'a Utf8Path,
52 pub lib: &'a Utf8Path,
53 pub root: &'a Utf8Path,
54 pub mode: Mode,
55 pub target: &'a TargetCodegenConfiguration,
56 pub config: &'a PackageConfig,
57 pub ids: UniqueIdGenerator,
58 pub write_metadata: bool,
59 pub perform_codegen: bool,
60 /// If set to false the compiler won't load and analyse any of the package's
61 /// modules and always succeed compilation returning no compile modules.
62 ///
63 /// Code generation is still carried out so that a root package will have an
64 /// entry point nonetheless.
65 ///
66 pub compile_modules: bool,
67 pub write_entrypoint: bool,
68 pub copy_native_files: bool,
69 pub compile_beam_bytecode: bool,
70 pub subprocess_stdio: Stdio,
71 pub target_support: TargetSupport,
72 pub cached_warnings: CachedWarnings,
73 pub check_module_conflicts: CheckModuleConflicts,
74}
75
76impl<'a, IO> PackageCompiler<'a, IO>
77where
78 IO: FileSystemReader + FileSystemWriter + CommandExecutor + BeamCompiler + Clone,
79{
80 pub fn new(
81 config: &'a PackageConfig,
82 mode: Mode,
83 root: &'a Utf8Path,
84 out: &'a Utf8Path,
85 lib: &'a Utf8Path,
86 target: &'a TargetCodegenConfiguration,
87 ids: UniqueIdGenerator,
88 io: IO,
89 ) -> Self {
90 Self {
91 io,
92 ids,
93 out,
94 lib,
95 root,
96 mode,
97 config,
98 target,
99 write_metadata: true,
100 perform_codegen: true,
101 compile_modules: true,
102 write_entrypoint: false,
103 copy_native_files: true,
104 compile_beam_bytecode: true,
105 subprocess_stdio: Stdio::Inherit,
106 target_support: TargetSupport::NotEnforced,
107 cached_warnings: CachedWarnings::Ignore,
108 check_module_conflicts: CheckModuleConflicts::DoNotCheck,
109 }
110 }
111
112 /// Compile the package.
113 /// Returns a list of modules that were compiled. Any modules that were read
114 /// from the cache will not be returned.
115 // TODO: return the cached modules.
116 pub fn compile(
117 mut self,
118 warnings: &WarningEmitter,
119 existing_modules: &mut im::HashMap<EcoString, type_::ModuleInterface>,
120 already_defined_modules: &mut im::HashMap<EcoString, DefinedModuleOrigin>,
121 stale_modules: &mut StaleTracker,
122 incomplete_modules: &mut HashSet<EcoString>,
123 telemetry: &dyn Telemetry,
124 ) -> Outcome<Compiled, Error> {
125 let span = tracing::info_span!("compile", package = %self.config.name.as_str());
126 let _enter = span.enter();
127
128 // Ensure that the package is compatible with this version of Gleam
129 if let Err(e) = self.config.check_gleam_compatibility() {
130 return e.into();
131 }
132
133 let artefact_directory = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
134 let codegen_required = if self.perform_codegen {
135 CodegenRequired::Yes
136 } else {
137 CodegenRequired::No
138 };
139
140 let loader = PackageLoader::new(
141 self.io.clone(),
142 self.ids.clone(),
143 self.mode,
144 self.root,
145 self.cached_warnings,
146 warnings,
147 codegen_required,
148 &artefact_directory,
149 self.target.target(),
150 &self.config.name,
151 stale_modules,
152 already_defined_modules,
153 incomplete_modules,
154 );
155
156 let loaded = if self.compile_modules {
157 match loader.run() {
158 Ok(loaded) => loaded,
159 Err(error) => return error.into(),
160 }
161 } else {
162 Loaded::empty()
163 };
164
165 let mut cached_module_names = Vec::new();
166
167 // Load the cached modules that have previously been compiled
168 for module in loaded.cached.into_iter() {
169 // Emit any cached warnings.
170 // Note that `self.cached_warnings` is set to `Ignore` (such as for
171 // dependency packages) then this field will not be populated.
172 if let Err(e) = self.emit_warnings(warnings, &module) {
173 return e.into();
174 }
175
176 cached_module_names.push(module.name.clone());
177
178 // Register the cached module so its type information etc can be
179 // used for compiling futher modules.
180 _ = existing_modules.insert(module.name.clone(), module);
181 }
182
183 if !loaded.to_compile.is_empty() {
184 // Print that work is being done
185 if self.perform_codegen {
186 telemetry.compiling_package(&self.config.name);
187 } else {
188 telemetry.checking_package(&self.config.name)
189 }
190 }
191
192 // Type check the modules that are new or have changed
193 tracing::info!(count=%loaded.to_compile.len(), "analysing_modules");
194 let outcome = analyse(
195 &self.config,
196 self.target.target(),
197 self.mode,
198 &self.ids,
199 loaded.to_compile,
200 existing_modules,
201 warnings,
202 self.target_support,
203 incomplete_modules,
204 );
205
206 let mut modules = match outcome {
207 Outcome::Ok(modules) => modules,
208 Outcome::PartialFailure(modules, error) => {
209 return Outcome::PartialFailure(
210 Compiled {
211 modules,
212 cached_module_names,
213 },
214 error,
215 );
216 }
217 Outcome::TotalFailure(error) => return Outcome::TotalFailure(error),
218 };
219
220 tracing::debug!("performing_code_generation");
221
222 // Inlining is currently disabled. See
223 // https://github.com/gleam-lang/gleam/pull/5010 for information.
224
225 // let modules = if self.perform_codegen {
226 // modules
227 // .into_iter()
228 // .map(|mut module| {
229 // module.ast = inline::module(module.ast, &existing_modules);
230 // module
231 // })
232 // .collect()
233 // } else {
234 // modules
235 // };
236
237 if let Err(error) = self.perform_codegen(&modules) {
238 return error.into();
239 }
240
241 if let Err(error) = self.encode_and_write_metadata(&mut modules) {
242 return error.into();
243 }
244
245 Outcome::Ok(Compiled {
246 modules,
247 cached_module_names,
248 })
249 }
250
251 fn compile_erlang_to_beam(
252 &mut self,
253 modules: &HashSet<Utf8PathBuf>,
254 ) -> Result<Vec<EcoString>, Error> {
255 if modules.is_empty() {
256 tracing::debug!("no_erlang_to_compile");
257 return Ok(Vec::new());
258 }
259
260 tracing::debug!("compiling_erlang");
261
262 self.io
263 .compile_beam(self.out, self.lib, modules, self.subprocess_stdio)
264 .map(|modules| modules.iter().map(|str| EcoString::from(str)).collect())
265 }
266
267 fn copy_project_native_files(
268 &mut self,
269 destination_dir: &Utf8Path,
270 to_compile_modules: &mut HashSet<Utf8PathBuf>,
271 ) -> Result<(), Error> {
272 tracing::debug!("copying_native_source_files");
273
274 // TODO: unit test
275 let priv_source = self.root.join("priv");
276 let priv_build = self.out.join("priv");
277 if self.io.is_directory(&priv_source) && !self.io.is_directory(&priv_build) {
278 tracing::debug!("linking_priv_to_build");
279 self.io.symlink_dir(&priv_source, &priv_build)?;
280 }
281
282 let copier = NativeFileCopier::new(
283 self.io.clone(),
284 self.root.clone(),
285 destination_dir,
286 self.check_module_conflicts,
287 );
288 let copied = copier.run()?;
289
290 to_compile_modules.extend(copied.to_compile.into_iter());
291
292 // If there are any Elixir files then we need to locate Elixir
293 // installed on this system for use in compilation.
294 if copied.any_elixir {
295 ElixirLibraries::make_available(
296 &self.io,
297 &self.lib.to_path_buf(),
298 self.subprocess_stdio,
299 )?;
300 }
301
302 Ok(())
303 }
304
305 fn encode_and_write_metadata(&mut self, modules: &mut [Module]) -> Result<()> {
306 if !self.write_metadata {
307 tracing::debug!("package_metadata_writing_disabled");
308 return Ok(());
309 }
310 if modules.is_empty() {
311 return Ok(());
312 }
313
314 let artefact_dir = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
315
316 tracing::debug!("writing_module_caches");
317 for module in modules {
318 let cache_files = CacheFiles::new(&artefact_dir, &module.name);
319
320 let result = if self.cached_warnings.should_use() {
321 metadata::encode(&module.ast.type_info)
322 } else {
323 // Dependency packages don't get warnings persisted as the
324 // programmer doesn't want to be told every time about warnings they
325 // cannot fix directly.
326 let warnings = std::mem::take(&mut module.ast.type_info.warnings);
327 let result = metadata::encode(&module.ast.type_info);
328 module.ast.type_info.warnings = warnings;
329 result
330 };
331
332 // Write cache file
333 let bytes = result.expect("Failed to serialise module cache");
334 self.io.write_bytes(&cache_files.cache_path, &bytes)?;
335
336 // Write cache metadata
337 let info = CacheMetadata {
338 mtime: module.mtime,
339 codegen_performed: self.perform_codegen,
340 dependencies: module.dependencies.clone(),
341 fingerprint: SourceFingerprint::new(&module.code),
342 line_numbers: module.ast.type_info.line_numbers.clone(),
343 };
344 self.io
345 .write_bytes(&cache_files.meta_path, &info.to_binary())?;
346 }
347 Ok(())
348 }
349
350 fn perform_codegen(&mut self, modules: &[Module]) -> Result<()> {
351 if !self.perform_codegen {
352 tracing::debug!("skipping_codegen");
353 return Ok(());
354 }
355
356 match self.target {
357 TargetCodegenConfiguration::JavaScript {
358 emit_typescript_definitions,
359 emit_source_maps,
360 prelude_location,
361 } => self.perform_javascript_codegen(
362 modules,
363 *emit_typescript_definitions,
364 *emit_source_maps,
365 prelude_location,
366 ),
367 TargetCodegenConfiguration::Erlang { app_file } => {
368 self.perform_erlang_codegen(modules, app_file.as_ref())
369 }
370 }
371 }
372
373 fn perform_erlang_codegen(
374 &mut self,
375 modules: &[Module],
376 app_file_config: Option<&ErlangAppCodegenConfiguration>,
377 ) -> Result<(), Error> {
378 let mut written = HashSet::new();
379 let build_dir = self.out.join(paths::ARTEFACT_DIRECTORY_NAME);
380 let include_dir = self.out.join("include");
381 let io = self.io.clone();
382
383 io.mkdir(&build_dir)?;
384
385 if self.copy_native_files {
386 self.copy_project_native_files(&build_dir, &mut written)?;
387 } else {
388 tracing::debug!("skipping_native_file_copying");
389 }
390
391 if self.compile_beam_bytecode && self.write_entrypoint {
392 self.render_erlang_entrypoint_module(&build_dir, &mut written)?;
393 } else {
394 tracing::debug!("skipping_entrypoint_generation");
395 }
396
397 // NOTE: This must come after `copy_project_native_files` to ensure that
398 // we overwrite any precompiled Erlang that was included in the Hex
399 // package. Otherwise we will build the potentially outdated precompiled
400 // version and not the newly compiled version.
401 Erlang::new(&build_dir, &include_dir).render(io.clone(), modules, self.root)?;
402
403 let native_modules: Vec<EcoString> = if self.compile_beam_bytecode {
404 written.extend(modules.iter().map(Module::compiled_erlang_path));
405 self.compile_erlang_to_beam(&written)?
406 } else {
407 tracing::debug!("skipping_erlang_bytecode_compilation");
408 Vec::new()
409 };
410
411 if let Some(config) = app_file_config {
412 ErlangApp::new(&self.out.join("ebin"), config).render(
413 io,
414 &self.config,
415 modules,
416 native_modules,
417 )?;
418 }
419 Ok(())
420 }
421
422 fn perform_javascript_codegen(
423 &mut self,
424 modules: &[Module],
425 typescript: bool,
426 sourcemap: bool,
427 prelude_location: &Utf8Path,
428 ) -> Result<(), Error> {
429 let mut written = HashSet::new();
430 let typescript = if typescript {
431 TypeScriptDeclarations::Emit
432 } else {
433 TypeScriptDeclarations::None
434 };
435 JavaScript::new(
436 &self.out,
437 typescript,
438 sourcemap,
439 prelude_location,
440 &self.root,
441 )
442 .render(&self.io, modules, self.stdlib_package())?;
443
444 if self.copy_native_files {
445 self.copy_project_native_files(&self.out, &mut written)?;
446 } else {
447 tracing::debug!("skipping_native_file_copying");
448 }
449
450 Ok(())
451 }
452
453 fn render_erlang_entrypoint_module(
454 &mut self,
455 out: &Utf8Path,
456 modules_to_compile: &mut HashSet<Utf8PathBuf>,
457 ) -> Result<(), Error> {
458 let name = format!("{name}@@main.erl", name = self.config.name);
459 let path = out.join(&name);
460
461 // If the entrypoint module has already been created then we don't need
462 // to write and compile it again.
463 if self.io.is_file(&path) {
464 tracing::debug!("erlang_entrypoint_already_exists");
465 return Ok(());
466 }
467
468 let template = ErlangEntrypointModule {
469 application: &self.config.name,
470 };
471 let module = template.render().expect("Erlang entrypoint rendering");
472 self.io.write(&path, &module)?;
473 let _ = modules_to_compile.insert(name.into());
474 tracing::debug!("erlang_entrypoint_written");
475 Ok(())
476 }
477
478 fn emit_warnings(
479 &self,
480 warnings: &WarningEmitter,
481 module: &type_::ModuleInterface,
482 ) -> Result<()> {
483 for warning in &module.warnings {
484 let src = self.io.read(&module.src_path)?;
485 warnings.emit(Warning::Type {
486 path: module.src_path.clone(),
487 src: src.into(),
488 warning: warning.clone(),
489 });
490 }
491
492 Ok(())
493 }
494
495 fn stdlib_package(&self) -> StdlibPackage {
496 if self.config.dependencies.contains_key("gleam_stdlib")
497 || self.config.dev_dependencies.contains_key("gleam_stdlib")
498 {
499 StdlibPackage::Present
500 } else {
501 StdlibPackage::Missing
502 }
503 }
504}
505
506#[derive(Debug, Clone, Copy, Eq, PartialEq)]
507pub enum StdlibPackage {
508 Present,
509 Missing,
510}
511
512fn analyse(
513 package_config: &PackageConfig,
514 target: Target,
515 mode: Mode,
516 ids: &UniqueIdGenerator,
517 mut parsed_modules: Vec<UncompiledModule>,
518 module_types: &mut im::HashMap<EcoString, type_::ModuleInterface>,
519 warnings: &WarningEmitter,
520 target_support: TargetSupport,
521 incomplete_modules: &mut HashSet<EcoString>,
522) -> Outcome<Vec<Module>, Error> {
523 let mut modules = Vec::with_capacity(parsed_modules.len() + 1);
524 let direct_dependencies = package_config.dependencies_for(mode).expect("Package deps");
525 let dev_dependencies = package_config.dev_dependencies.keys().cloned().collect();
526
527 // Insert the prelude
528 // DUPE: preludeinsertion
529 // TODO: Currently we do this here and also in the tests. It would be better
530 // to have one place where we create all this required state for use in each
531 // place.
532 let _ = module_types.insert(PRELUDE_MODULE_NAME.into(), type_::build_prelude(ids));
533
534 for UncompiledModule {
535 name,
536 code,
537 ast,
538 path,
539 mtime,
540 origin,
541 package,
542 dependencies,
543 extra,
544 } in parsed_modules
545 {
546 tracing::debug!(module = ?name, "Type checking");
547
548 let line_numbers = LineNumbers::new(&code);
549
550 let analysis = crate::analyse::ModuleAnalyzerConstructor {
551 target,
552 ids,
553 origin,
554 importable_modules: module_types,
555 warnings: &TypeWarningEmitter::new(path.clone(), code.clone(), warnings.clone()),
556 direct_dependencies: &direct_dependencies,
557 dev_dependencies: &dev_dependencies,
558 target_support,
559 package_config,
560 }
561 .infer_module(ast, line_numbers, path.clone());
562
563 match analysis {
564 Outcome::Ok(ast) => {
565 // Module has compiled successfully. Make sure it isn't marked as incomplete.
566 let _ = incomplete_modules.remove(&name.clone());
567
568 let mut module = Module {
569 dependencies,
570 origin,
571 extra,
572 mtime,
573 name,
574 code,
575 ast,
576 input_path: path,
577 };
578 module.attach_doc_and_module_comments();
579
580 // Register the types from this module so they can be imported into
581 // other modules.
582 let _ = module_types.insert(module.name.clone(), module.ast.type_info.clone());
583
584 // Check for empty modules and emit warning
585 // Only emit the empty module warning if the module has no definitions at all.
586 // Modules with only private definitions already emit their own warnings.
587 if module_types
588 .get(&module.name)
589 .map(|interface| interface.values.is_empty() && interface.types.is_empty())
590 .unwrap_or(false)
591 {
592 warnings.emit(crate::warning::Warning::EmptyModule {
593 path: module.input_path.clone(),
594 name: module.name.clone(),
595 });
596 }
597
598 // Register the successfully type checked module data so that it can be
599 // used for code generation and in the language server.
600 modules.push(module);
601 }
602
603 Outcome::PartialFailure(ast, errors) => {
604 let error = Error::Type {
605 names: Box::new(ast.names.clone()),
606 path: path.clone(),
607 src: code.clone(),
608 errors,
609 };
610 // Mark as incomplete so that this module isn't reloaded from cache.
611 let _ = incomplete_modules.insert(name.clone());
612 // Register the partially type checked module data so that it can be
613 // used in the language server.
614 let mut module = Module {
615 dependencies,
616 origin,
617 extra,
618 mtime,
619 name,
620 code,
621 ast,
622 input_path: path,
623 };
624 module.attach_doc_and_module_comments();
625
626 let _ = module_types.insert(module.ast.name.clone(), module.ast.type_info.clone());
627
628 modules.push(module);
629 // WARNING: This cannot be used for code generation as the code has errors.
630 return Outcome::PartialFailure(modules, error);
631 }
632
633 Outcome::TotalFailure(errors) => {
634 return Outcome::TotalFailure(Error::Type {
635 names: Default::default(),
636 path: path.clone(),
637 src: code.clone(),
638 errors,
639 });
640 }
641 };
642 }
643
644 Outcome::Ok(modules)
645}
646
647#[derive(Debug)]
648pub(crate) enum Input {
649 New(UncompiledModule),
650 Cached(CachedModule),
651}
652
653impl Input {
654 pub fn name(&self) -> &EcoString {
655 match self {
656 Input::New(m) => &m.name,
657 Input::Cached(m) => &m.name,
658 }
659 }
660
661 pub fn source_path(&self) -> &Utf8Path {
662 match self {
663 Input::New(m) => &m.path,
664 Input::Cached(m) => &m.source_path,
665 }
666 }
667
668 pub fn dependencies(&self) -> Vec<EcoString> {
669 match self {
670 Input::New(m) => m.dependencies.iter().map(|(n, _)| n.clone()).collect(),
671 Input::Cached(m) => m.dependencies.iter().map(|(n, _)| n.clone()).collect(),
672 }
673 }
674
675 /// Returns `true` if the input is [`New`].
676 ///
677 /// [`New`]: Input::New
678 #[must_use]
679 pub(crate) fn is_new(&self) -> bool {
680 matches!(self, Self::New(..))
681 }
682
683 /// Returns `true` if the input is [`Cached`].
684 ///
685 /// [`Cached`]: Input::Cached
686 #[must_use]
687 pub(crate) fn is_cached(&self) -> bool {
688 matches!(self, Self::Cached(..))
689 }
690}
691
692#[derive(Debug)]
693pub(crate) struct CachedModule {
694 pub name: EcoString,
695 pub origin: Origin,
696 pub dependencies: Vec<(EcoString, SrcSpan)>,
697 pub source_path: Utf8PathBuf,
698 pub line_numbers: LineNumbers,
699}
700
701#[derive(Debug, serde::Serialize, serde::Deserialize)]
702pub(crate) struct CacheMetadata {
703 pub mtime: SystemTime,
704 pub codegen_performed: bool,
705 pub dependencies: Vec<(EcoString, SrcSpan)>,
706 pub fingerprint: SourceFingerprint,
707 pub line_numbers: LineNumbers,
708}
709
710impl CacheMetadata {
711 pub fn to_binary(&self) -> Vec<u8> {
712 bincode::serde::encode_to_vec(self, bincode::config::legacy())
713 .expect("Serializing cache info")
714 }
715
716 pub fn from_binary(bytes: &[u8]) -> Result<Self, String> {
717 match bincode::serde::decode_from_slice(bytes, bincode::config::legacy()) {
718 Ok((data, _)) => Ok(data),
719 Err(e) => Err(e.to_string()),
720 }
721 }
722}
723
724#[derive(Debug, Default, PartialEq, Eq)]
725pub(crate) struct Loaded {
726 pub to_compile: Vec<UncompiledModule>,
727 pub cached: Vec<type_::ModuleInterface>,
728}
729
730impl Loaded {
731 fn empty() -> Self {
732 Self {
733 to_compile: vec![],
734 cached: vec![],
735 }
736 }
737}
738
739#[derive(Debug, PartialEq, Eq)]
740pub(crate) struct UncompiledModule {
741 pub path: Utf8PathBuf,
742 pub name: EcoString,
743 pub code: EcoString,
744 pub mtime: SystemTime,
745 pub origin: Origin,
746 pub package: EcoString,
747 pub dependencies: Vec<(EcoString, SrcSpan)>,
748 pub ast: UntypedModule,
749 pub extra: ModuleExtra,
750}
751
752#[derive(Template)]
753#[template(path = "gleam@@main.erl", escape = "none")]
754struct ErlangEntrypointModule<'a> {
755 application: &'a str,
756}
757
758#[derive(Debug, Clone, Copy)]
759pub enum CachedWarnings {
760 Use,
761 Ignore,
762}
763impl CachedWarnings {
764 pub(crate) fn should_use(&self) -> bool {
765 match self {
766 CachedWarnings::Use => true,
767 CachedWarnings::Ignore => false,
768 }
769 }
770}
771
772#[derive(Debug, Clone, Copy)]
773pub enum CheckModuleConflicts {
774 Check,
775 DoNotCheck,
776}
777impl CheckModuleConflicts {
778 pub(crate) fn should_check(&self) -> bool {
779 match self {
780 CheckModuleConflicts::Check => true,
781 CheckModuleConflicts::DoNotCheck => false,
782 }
783 }
784}