nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
25 kB
688 lines
1use crate::{
2 Error, Result, Warning,
3 analyse::TargetSupport,
4 build::{
5 Mode, Module, Origin, Package, Target,
6 package_compiler::{self, PackageCompiler},
7 package_loader::StaleTracker,
8 project_compiler,
9 telemetry::Telemetry,
10 },
11 codegen::{self, ErlangApp},
12 config::PackageConfig,
13 dep_tree,
14 error::{FileIoAction, FileKind, ShellCommandFailureReason},
15 io::{BeamCompiler, Command, CommandExecutor, FileSystemReader, FileSystemWriter, Stdio},
16 manifest::{ManifestPackage, ManifestPackageSource},
17 metadata,
18 paths::{self, ProjectPaths},
19 type_::{self, ModuleFunction},
20 uid::UniqueIdGenerator,
21 version::COMPILER_VERSION,
22 warning::{self, WarningEmitter, WarningEmitterIO},
23};
24use ecow::EcoString;
25use hexpm::version::Version;
26use itertools::Itertools;
27use pubgrub::range::Range;
28use std::{
29 cmp,
30 collections::{HashMap, HashSet},
31 fmt::Write,
32 io::BufReader,
33 rc::Rc,
34 sync::Arc,
35 time::Instant,
36};
37
38use super::{
39 Codegen, Compile, ErlangAppCodegenConfiguration, Outcome,
40 elixir_libraries::ElixirLibraries,
41 package_compiler::{CachedWarnings, Compiled},
42};
43
44use camino::{Utf8Path, Utf8PathBuf};
45
46// On Windows we have to call rebar3 via a little wrapper script.
47//
48#[cfg(not(target_os = "windows"))]
49const REBAR_EXECUTABLE: &str = "rebar3";
50#[cfg(target_os = "windows")]
51const REBAR_EXECUTABLE: &str = "rebar3.cmd";
52
53#[cfg(not(target_os = "windows"))]
54const ELIXIR_EXECUTABLE: &str = "elixir";
55#[cfg(target_os = "windows")]
56const ELIXIR_EXECUTABLE: &str = "elixir.bat";
57
58#[derive(Debug)]
59pub struct Options {
60 pub mode: Mode,
61 pub target: Option<Target>,
62 pub compile: Compile,
63 pub codegen: Codegen,
64 pub warnings_as_errors: bool,
65 pub root_target_support: TargetSupport,
66 pub no_print_progress: bool,
67}
68
69#[derive(Debug)]
70pub struct Built {
71 pub root_package: Package,
72 pub module_interfaces: im::HashMap<EcoString, type_::ModuleInterface>,
73 compiled_dependency_modules: Vec<Module>,
74}
75
76impl Built {
77 pub fn get_main_function(
78 &self,
79 module: &EcoString,
80 target: Target,
81 ) -> Result<ModuleFunction, Error> {
82 match self.module_interfaces.get(module) {
83 Some(module_data) => module_data.get_main_function(target),
84 None => Err(Error::ModuleDoesNotExist {
85 module: module.clone(),
86 suggestion: None,
87 }),
88 }
89 }
90
91 pub fn minimum_required_version(&self) -> Version {
92 self.module_interfaces
93 .values()
94 .map(|interface| &interface.minimum_required_version)
95 .reduce(|one_version, other_version| cmp::max(one_version, other_version))
96 .map(|minimum_required_version| minimum_required_version.clone())
97 .unwrap_or(Version::new(0, 1, 0))
98 }
99}
100
101#[derive(Debug)]
102pub struct ProjectCompiler<IO> {
103 // The gleam.toml config for the root package of the project
104 pub(crate) config: PackageConfig,
105 pub(crate) packages: HashMap<String, ManifestPackage>,
106 importable_modules: im::HashMap<EcoString, type_::ModuleInterface>,
107 defined_modules: im::HashMap<EcoString, Utf8PathBuf>,
108 stale_modules: StaleTracker,
109 /// The set of modules that have had partial compilation done since the last
110 /// successful compilation.
111 incomplete_modules: HashSet<EcoString>,
112 warnings: WarningEmitter,
113 telemetry: &'static dyn Telemetry,
114 options: Options,
115 paths: ProjectPaths,
116 ids: UniqueIdGenerator,
117 pub(crate) io: IO,
118 /// We may want to silence subprocess stdout if we are running in LSP mode.
119 /// The language server talks over stdio so printing would break that.
120 pub subprocess_stdio: Stdio,
121}
122
123// TODO: test that tests cannot be imported into src
124// TODO: test that dep cycles are not allowed between packages
125
126impl<IO> ProjectCompiler<IO>
127where
128 IO: CommandExecutor + FileSystemWriter + FileSystemReader + BeamCompiler + Clone,
129{
130 pub fn new(
131 config: PackageConfig,
132 options: Options,
133 packages: Vec<ManifestPackage>,
134 telemetry: &'static dyn Telemetry,
135 warning_emitter: Rc<dyn WarningEmitterIO>,
136 paths: ProjectPaths,
137 io: IO,
138 ) -> Self {
139 let packages = packages
140 .into_iter()
141 .map(|p| (p.name.to_string(), p))
142 .collect();
143
144 Self {
145 importable_modules: im::HashMap::new(),
146 defined_modules: im::HashMap::new(),
147 stale_modules: StaleTracker::default(),
148 incomplete_modules: HashSet::new(),
149 ids: UniqueIdGenerator::new(),
150 warnings: WarningEmitter::new(warning_emitter),
151 subprocess_stdio: Stdio::Inherit,
152 telemetry,
153 packages,
154 options,
155 config,
156 paths,
157 io,
158 }
159 }
160
161 pub fn get_importable_modules(&self) -> &im::HashMap<EcoString, type_::ModuleInterface> {
162 &self.importable_modules
163 }
164
165 pub fn mode(&self) -> Mode {
166 self.options.mode
167 }
168
169 pub fn target(&self) -> Target {
170 self.options.target.unwrap_or(self.config.target)
171 }
172
173 /// Compiles all packages in the project and returns the compiled
174 /// information from the root package
175 pub fn compile(mut self) -> Result<Built> {
176 // We make sure the stale module tracker is empty before we start, to
177 // avoid mistakenly thinking a module is stale due to outdated state
178 // from a previous build. A ProjectCompiler instance is re-used by the
179 // LSP engine so state could be reused if we don't reset it.
180 self.stale_modules.empty();
181
182 // Each package may specify a Gleam version that it supports, so we
183 // verify that this version is appropriate.
184 self.check_gleam_version()?;
185
186 // The JavaScript target requires a prelude module to be written.
187 self.write_prelude()?;
188
189 // Dependencies are compiled first.
190 let compiled_dependency_modules = self.compile_dependencies()?;
191
192 // We reset the warning count as we don't want to fail the build if a
193 // dependency has warnings, only if the root package does.
194 self.warnings.reset_count();
195
196 let root_package = self.compile_root_package().into_result()?;
197
198 // TODO: test
199 if self.options.warnings_as_errors && self.warnings.count() > 0 {
200 return Err(Error::ForbiddenWarnings {
201 count: self.warnings.count(),
202 });
203 }
204
205 Ok(Built {
206 root_package,
207 module_interfaces: self.importable_modules,
208 compiled_dependency_modules,
209 })
210 }
211
212 pub fn compile_root_package(&mut self) -> Outcome<Package, Error> {
213 let config = self.config.clone();
214 self.compile_gleam_package(&config, true, self.paths.root().to_path_buf())
215 .map(
216 |Compiled {
217 modules,
218 module_names,
219 }| Package {
220 config,
221 modules,
222 module_names,
223 },
224 )
225 }
226
227 /// Checks that version file found in the build directory matches the
228 /// current version of gleam. If not, we will clear the build directory
229 /// before continuing. This will ensure that upgrading gleam will not leave
230 /// one with confusing or hard to debug states.
231 pub fn check_gleam_version(&self) -> Result<(), Error> {
232 let build_path = self
233 .paths
234 .build_directory_for_target(self.mode(), self.target());
235 let version_path = self.paths.build_gleam_version(self.mode(), self.target());
236 if self.io.is_file(&version_path) {
237 let version = self.io.read(&version_path)?;
238 if version == COMPILER_VERSION {
239 return Ok(());
240 }
241 }
242
243 // Either file is missing our the versions do not match. Time to rebuild
244 tracing::info!("removing_build_state_from_different_gleam_version");
245 self.io.delete_directory(&build_path)?;
246
247 // Recreate build directory with new updated version file
248 self.io.mkdir(&build_path)?;
249 self.io
250 .write(&version_path, COMPILER_VERSION)
251 .map_err(|e| Error::FileIo {
252 action: FileIoAction::WriteTo,
253 kind: FileKind::File,
254 path: version_path,
255 err: Some(e.to_string()),
256 })
257 }
258
259 pub fn compile_dependencies(&mut self) -> Result<Vec<Module>, Error> {
260 let sequence = order_packages(&self.packages)?;
261 let mut modules = vec![];
262
263 for name in sequence {
264 let compiled = self.load_cache_or_compile_package(&name)?;
265 modules.extend(compiled);
266 }
267
268 Ok(modules)
269 }
270
271 fn write_prelude(&self) -> Result<()> {
272 // Only the JavaScript target has a prelude to write.
273 if !self.target().is_javascript() {
274 return Ok(());
275 }
276
277 let build = self
278 .paths
279 .build_directory_for_target(self.mode(), self.target());
280
281 // Write the JavaScript prelude
282 let path = build.join("prelude.mjs");
283 if !self.io.is_file(&path) {
284 self.io.write(&path, crate::javascript::PRELUDE)?;
285 }
286
287 // Write the TypeScript prelude, if asked for
288 if self.config.javascript.typescript_declarations {
289 let path = build.join("prelude.d.mts");
290 if !self.io.is_file(&path) {
291 self.io.write(&path, crate::javascript::PRELUDE_TS_DEF)?;
292 }
293 }
294
295 Ok(())
296 }
297
298 fn load_cache_or_compile_package(&mut self, name: &str) -> Result<Vec<Module>, Error> {
299 // TODO: We could remove this clone if we split out the compilation of
300 // packages into their own classes and then only mutate self after we no
301 // longer need to have the package borrowed from self.packages.
302 let package = self.packages.get(name).expect("Missing package").clone();
303 let result = match usable_build_tools(&package)?.as_slice() {
304 &[BuildTool::Gleam] => self.compile_gleam_dep_package(&package),
305 &[BuildTool::Rebar3] => self.compile_rebar3_dep_package(&package).map(|_| vec![]),
306 &[BuildTool::Mix] => self.compile_mix_dep_package(&package).map(|_| vec![]),
307 &[BuildTool::Mix, BuildTool::Rebar3] => self
308 .compile_mix_dep_package(&package)
309 .or_else(|_| self.compile_rebar3_dep_package(&package))
310 .map(|_| vec![]),
311 _ => {
312 return Err(Error::UnsupportedBuildTool {
313 package: package.name.to_string(),
314 build_tools: package.build_tools.clone(),
315 });
316 }
317 };
318
319 // TODO: test. This one is not covered by the integration tests.
320 if result.is_err() {
321 tracing::debug!(package=%name, "removing_failed_build");
322 let path = self.paths.build_directory_for_package(
323 self.mode(),
324 self.target(),
325 package.application_name(),
326 );
327 self.io.delete_directory(&path)?;
328 }
329
330 result
331 }
332
333 // TODO: extract and unit test
334 fn compile_rebar3_dep_package(&mut self, package: &ManifestPackage) -> Result<(), Error> {
335 let application_name = package.application_name();
336 let package_name = &package.name;
337 let mode = self.mode();
338 let target = self.target();
339
340 let package_build = self
341 .paths
342 .build_directory_for_package(mode, target, application_name);
343
344 // TODO: test
345 if self.io.is_directory(&package_build) {
346 tracing::debug!(%package_name, "using_precompiled_rebar3_package");
347 return Ok(());
348 }
349
350 // TODO: test
351 if !self.options.codegen.should_codegen(false) {
352 tracing::debug!(%package_name, "skipping_rebar3_build_as_codegen_disabled");
353 return Ok(());
354 }
355
356 // TODO: test
357 if target != Target::Erlang {
358 tracing::debug!(%package_name, "skipping_rebar3_build_for_non_erlang_target");
359 return Ok(());
360 }
361
362 // Print that work is being done
363 self.telemetry.compiling_package(package_name);
364
365 let package = self.paths.build_packages_package(package_name);
366 let build_packages = self.paths.build_directory_for_target(mode, target);
367 let ebins = self.paths.build_packages_ebins_glob(mode, target);
368 let rebar3_path = |path: &Utf8Path| format!("../{}", path);
369
370 tracing::debug!("copying_package_to_build");
371 self.io.mkdir(&package_build)?;
372 self.io.copy_dir(&package, &package_build)?;
373
374 let env = vec![
375 ("ERL_LIBS".to_string(), "../*/ebin".to_string()),
376 (
377 "REBAR_BARE_COMPILER_OUTPUT_DIR".to_string(),
378 package_build.to_string(),
379 ),
380 ("REBAR_PROFILE".to_string(), "prod".to_string()),
381 ("REBAR_SKIP_PROJECT_PLUGINS".to_string(), "true".to_string()),
382 ("TERM".to_string(), "dumb".to_string()),
383 ];
384 let args = vec![
385 "bare".into(),
386 "compile".into(),
387 "--paths".into(),
388 "../*/ebin".into(),
389 ];
390
391 let status = self.io.exec(Command {
392 program: REBAR_EXECUTABLE.into(),
393 args,
394 env,
395 cwd: Some(package_build),
396 stdio: self.subprocess_stdio,
397 })?;
398
399 if status == 0 {
400 Ok(())
401 } else {
402 Err(Error::ShellCommand {
403 program: "rebar3".into(),
404 reason: ShellCommandFailureReason::Unknown,
405 })
406 }
407 }
408
409 fn compile_mix_dep_package(&mut self, package: &ManifestPackage) -> Result<(), Error> {
410 let application_name = package.application_name();
411 let package_name = &package.name;
412 let mode = self.mode();
413 let target = self.target();
414 let mix_target = "prod";
415
416 let dest = self
417 .paths
418 .build_directory_for_package(mode, target, application_name);
419
420 // TODO: test
421 if self.io.is_directory(&dest) {
422 tracing::debug!(%package_name, "using_precompiled_mix_package");
423 return Ok(());
424 }
425
426 // TODO: test
427 if !self.options.codegen.should_codegen(false) {
428 tracing::debug!(%package_name, "skipping_mix_build_as_codegen_disabled");
429 return Ok(());
430 }
431
432 // TODO: test
433 if target != Target::Erlang {
434 tracing::debug!(%package_name, "skipping_mix_build_for_non_erlang_target");
435 return Ok(());
436 }
437
438 // Print that work is being done
439 self.telemetry.compiling_package(package_name);
440
441 let build_dir = self.paths.build_directory_for_target(mode, target);
442 let project_dir = self.paths.build_packages_package(package_name);
443 let mix_build_dir = project_dir.join("_build").join(mix_target);
444 let mix_build_lib_dir = mix_build_dir.join("lib");
445 let up = paths::unnest(&project_dir);
446 let mix_path = |path: &Utf8Path| up.join(path).to_string();
447 let ebins = self.paths.build_packages_ebins_glob(mode, target);
448
449 // Elixir core libs must be loaded
450 ElixirLibraries::make_available(&self.io, &build_dir, self.subprocess_stdio)?;
451
452 // Prevent Mix.Compilers.ApplicationTracer warnings
453 // mix would make this if it didn't exist, but we make it anyway as
454 // we need to link the compiled dependencies into there
455 self.io.mkdir(&mix_build_lib_dir)?;
456 let deps = &package.requirements;
457 for dep in deps {
458 // TODO: unit test
459 let dep_source = build_dir.join(dep.as_str());
460 let dep_dest = mix_build_lib_dir.join(dep.as_str());
461 if self.io.is_directory(&dep_source) && !self.io.is_directory(&dep_dest) {
462 tracing::debug!("linking_{}_to_build", dep);
463 self.io.symlink_dir(&dep_source, &dep_dest)?;
464 }
465 }
466
467 let env = vec![
468 ("MIX_BUILD_PATH".to_string(), mix_path(&mix_build_dir)),
469 ("MIX_ENV".to_string(), mix_target.to_string()),
470 ("MIX_QUIET".to_string(), "1".to_string()),
471 ("TERM".to_string(), "dumb".to_string()),
472 ];
473 let args = vec![
474 "-pa".to_string(),
475 mix_path(&ebins),
476 "-S".to_string(),
477 "mix".to_string(),
478 "compile".to_string(),
479 "--no-deps-check".to_string(),
480 "--no-load-deps".to_string(),
481 "--no-protocol-consolidation".to_string(),
482 ];
483
484 let status = self.io.exec(Command {
485 program: ELIXIR_EXECUTABLE.into(),
486 args,
487 env,
488 cwd: Some(project_dir),
489 stdio: self.subprocess_stdio,
490 })?;
491
492 if status == 0 {
493 // TODO: unit test
494 let source = mix_build_dir.join("lib").join(application_name.as_str());
495 if self.io.is_directory(&source) && !self.io.is_directory(&dest) {
496 tracing::debug!("linking_{}_to_build", application_name);
497 self.io.symlink_dir(&source, &dest)?;
498 }
499 Ok(())
500 } else {
501 Err(Error::ShellCommand {
502 program: "mix".into(),
503 reason: ShellCommandFailureReason::Unknown,
504 })
505 }
506 }
507
508 fn compile_gleam_dep_package(
509 &mut self,
510 package: &ManifestPackage,
511 ) -> Result<Vec<Module>, Error> {
512 // TODO: Test
513 let package_root = match &package.source {
514 // If the path is relative it is relative to the root of the
515 // project, not to the current working directory. The language server
516 // could have the working directory and the project root in different
517 // places.
518 ManifestPackageSource::Local { path } if path.is_relative() => {
519 self.io.canonicalise(&self.paths.root().join(path))?
520 }
521
522 // If the path is absolute we can use it as-is.
523 ManifestPackageSource::Local { path } => path.clone(),
524
525 // Hex and Git packages are downloaded into the project's build
526 // directory.
527 ManifestPackageSource::Git { .. } | ManifestPackageSource::Hex { .. } => {
528 self.paths.build_packages_package(&package.name)
529 }
530 };
531 let config_path = package_root.join("gleam.toml");
532 let config = PackageConfig::read(config_path, &self.io)?;
533 self.compile_gleam_package(&config, false, package_root)
534 .into_result()
535 .map(|compiled| compiled.modules)
536 }
537
538 fn compile_gleam_package(
539 &mut self,
540 config: &PackageConfig,
541 is_root: bool,
542 root_path: Utf8PathBuf,
543 ) -> Outcome<Compiled, Error> {
544 let out_path =
545 self.paths
546 .build_directory_for_package(self.mode(), self.target(), &config.name);
547 let lib_path = self
548 .paths
549 .build_directory_for_target(self.mode(), self.target());
550 let mode = if is_root { self.mode() } else { Mode::Prod };
551 let target = match self.target() {
552 Target::Erlang => {
553 let package_name_overrides = self
554 .packages
555 .values()
556 .flat_map(|p| {
557 let overriden = p.otp_app.as_ref()?;
558 Some((p.name.clone(), overriden.clone()))
559 })
560 .collect();
561 super::TargetCodegenConfiguration::Erlang {
562 app_file: Some(ErlangAppCodegenConfiguration {
563 include_dev_deps: is_root && self.mode().includes_dev_dependencies(),
564 package_name_overrides,
565 }),
566 }
567 }
568
569 Target::JavaScript => super::TargetCodegenConfiguration::JavaScript {
570 emit_typescript_definitions: self.config.javascript.typescript_declarations,
571 // This path is relative to each package output directory
572 prelude_location: Utf8PathBuf::from("../prelude.mjs"),
573 },
574 };
575
576 let mut compiler = PackageCompiler::new(
577 config,
578 mode,
579 &root_path,
580 &out_path,
581 &lib_path,
582 &target,
583 self.ids.clone(),
584 self.io.clone(),
585 );
586 compiler.write_metadata = true;
587 compiler.write_entrypoint = is_root;
588 compiler.perform_codegen = self.options.codegen.should_codegen(is_root);
589 compiler.compile_beam_bytecode = self.options.codegen.should_codegen(is_root);
590 compiler.compile_modules = !(self.options.compile == Compile::DepsOnly && is_root);
591 compiler.subprocess_stdio = self.subprocess_stdio;
592 compiler.target_support = if is_root {
593 // When compiling the root package it is context specific as to whether we need to
594 // enforce that all functions have an implementation for the current target.
595 // Typically we do, but if we are using `gleam run -m $module` to run a module that
596 // belongs to a dependency we don't need to enforce this as we don't want to fail
597 // compilation. It's impossible for a dependecy module to call functions from the root
598 // package, so it's OK if they could not be compiled.
599 self.options.root_target_support
600 } else {
601 // When compiling dependencies we don't enforce that all functions have an
602 // implementation for the current target. It is OK if they have APIs that are
603 // unaccessible so long as they are not used by the root package.
604 TargetSupport::NotEnforced
605 };
606 compiler.cached_warnings = if is_root {
607 CachedWarnings::Use
608 } else {
609 CachedWarnings::Ignore
610 };
611
612 // Compile project to Erlang or JavaScript source code
613 compiler.compile(
614 &mut self.warnings,
615 &mut self.importable_modules,
616 &mut self.defined_modules,
617 &mut self.stale_modules,
618 &mut self.incomplete_modules,
619 self.telemetry,
620 )
621 }
622}
623
624fn order_packages(packages: &HashMap<String, ManifestPackage>) -> Result<Vec<EcoString>, Error> {
625 dep_tree::toposort_deps(
626 packages
627 .values()
628 // Making sure that the package order is deterministic, to prevent different
629 // compilations of the same project compiling in different orders. This could impact
630 // any bugged outcomes, though not any where the compiler is working correctly, so it's
631 // mostly to aid debugging.
632 .sorted_by(|a, b| a.name.cmp(&b.name))
633 .map(|package| {
634 (
635 package.name.as_str().into(),
636 package
637 .requirements
638 .iter()
639 .map(|r| EcoString::from(r.as_ref()))
640 .collect(),
641 )
642 })
643 .collect(),
644 )
645 .map_err(convert_deps_tree_error)
646}
647
648fn convert_deps_tree_error(e: dep_tree::Error) -> Error {
649 match e {
650 dep_tree::Error::Cycle(packages) => Error::PackageCycle { packages },
651 }
652}
653
654#[derive(Debug, PartialEq, Clone, Copy)]
655pub(crate) enum BuildTool {
656 Gleam,
657 Rebar3,
658 Mix,
659}
660
661/// Determine the build tool we should use to build this package
662pub(crate) fn usable_build_tools(package: &ManifestPackage) -> Result<Vec<BuildTool>, Error> {
663 let mut rebar3_present = false;
664 let mut mix_present = false;
665
666 for tool in &package.build_tools {
667 match tool.as_str() {
668 "gleam" => return Ok(vec![BuildTool::Gleam]),
669 "rebar" => rebar3_present = true,
670 "rebar3" => rebar3_present = true,
671 "mix" => mix_present = true,
672 _ => (),
673 }
674 }
675
676 if mix_present && rebar3_present {
677 return Ok(vec![BuildTool::Mix, BuildTool::Rebar3]);
678 } else if mix_present {
679 return Ok(vec![BuildTool::Mix]);
680 } else if rebar3_present {
681 return Ok(vec![BuildTool::Rebar3]);
682 }
683
684 Err(Error::UnsupportedBuildTool {
685 package: package.name.to_string(),
686 build_tools: package.build_tools.clone(),
687 })
688}