nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
59 kB
1757 lines
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2021 The Gleam contributors
3
4mod dependency_manager;
5
6use std::{
7 cell::RefCell,
8 collections::{HashMap, HashSet},
9 io::ErrorKind,
10 process::Command,
11 rc::Rc,
12 time::Instant,
13};
14
15use camino::{Utf8Path, Utf8PathBuf};
16use ecow::{EcoString, eco_format};
17use flate2::read::GzDecoder;
18use gleam_core::{
19 Error, Result,
20 build::{Mode, SourceFingerprint, Target, Telemetry},
21 config::PackageConfig,
22 dependency::{self, PackageFetchError},
23 error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
24 hex::{self, HEXPM_PUBLIC_KEY},
25 io::{HttpClient as _, TarUnpacker, WrappedReader},
26 manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, PackageChanges},
27 paths::ProjectPaths,
28 requirement::Requirement,
29};
30use hexpm::version::Version;
31use itertools::Itertools;
32use same_file::is_same_file;
33use strum::IntoEnumIterator;
34
35pub use dependency_manager::DependencyManagerConfig;
36
37#[cfg(test)]
38mod tests;
39
40use crate::{
41 TreeOptions,
42 build_lock::{BuildLock, Guard},
43 cli,
44 fs::{self, ProjectIO},
45 http::HttpClient,
46 text_layout::space_table,
47};
48
49struct Symbols {
50 down: &'static str,
51 tee: &'static str,
52 ell: &'static str,
53 right: &'static str,
54}
55
56static UTF8_SYMBOLS: Symbols = Symbols {
57 down: "│",
58 tee: "├",
59 ell: "└",
60 right: "─",
61};
62
63/// When set to `Yes`, the cli will check for major version updates of direct dependencies and
64/// print them to the console if the major versions are not upgradeable due to constraints.
65#[derive(Debug, Clone, Copy)]
66pub enum CheckMajorVersions {
67 Yes,
68 No,
69}
70
71pub fn list(paths: &ProjectPaths) -> Result<()> {
72 let (_, manifest) = get_manifest_details(paths)?;
73 list_manifest_packages(std::io::stdout(), manifest)
74}
75
76pub fn tree(paths: &ProjectPaths, options: TreeOptions) -> Result<()> {
77 let (config, manifest) = get_manifest_details(paths)?;
78
79 // Initialize the root package since it is not part of the manifest
80 let root_package = ManifestPackage {
81 build_tools: vec![],
82 name: config.name.clone(),
83 requirements: config.all_direct_dependencies()?.keys().cloned().collect(),
84 version: config.version.clone(),
85 source: ManifestPackageSource::Local {
86 path: paths.root().to_path_buf(),
87 },
88 otp_app: None,
89 };
90
91 // Get the manifest packages and add the root package to the vec
92 let mut packages = manifest.packages.iter().cloned().collect_vec();
93 packages.push(root_package);
94
95 list_package_and_dependencies_tree(std::io::stdout(), options, packages.clone(), config.name)
96}
97
98fn get_manifest_details(paths: &ProjectPaths) -> Result<(PackageConfig, Manifest)> {
99 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
100 let config = crate::config::root_config(paths)?;
101 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
102 let dependency_manager = DependencyManagerConfig {
103 use_manifest: UseManifest::Yes,
104 check_major_versions: CheckMajorVersions::No,
105 }
106 .into_dependency_manager(
107 runtime.handle().clone(),
108 package_fetcher,
109 cli::Reporter::new(),
110 Mode::Dev,
111 );
112 let manifest = dependency_manager
113 .resolve_versions(paths, &config, Vec::new())?
114 .manifest;
115 Ok((config, manifest))
116}
117
118fn list_manifest_packages<W: std::io::Write>(mut buffer: W, manifest: Manifest) -> Result<()> {
119 let packages = manifest
120 .packages
121 .into_iter()
122 .map(|package| vec![package.name.to_string(), package.version.to_string()])
123 .collect_vec();
124 let out = space_table(&["Package", "Version"], packages);
125
126 write!(buffer, "{out}").map_err(|e| Error::StandardIo {
127 action: StandardIoAction::Write,
128 err: Some(e.kind()),
129 })
130}
131
132fn list_package_and_dependencies_tree<W: std::io::Write>(
133 mut buffer: W,
134 options: TreeOptions,
135 packages: Vec<ManifestPackage>,
136 root_package_name: EcoString,
137) -> Result<()> {
138 let mut invert = false;
139
140 let package: Option<&ManifestPackage> = if let Some(input_package_name) = options.package {
141 packages.iter().find(|p| p.name == input_package_name)
142 } else if let Some(input_package_name) = options.invert {
143 invert = true;
144 packages.iter().find(|p| p.name == input_package_name)
145 } else {
146 packages.iter().find(|p| p.name == root_package_name)
147 };
148
149 if let Some(package) = package {
150 let tree = Vec::from([eco_format!("{0} v{1}", package.name, package.version)]);
151 let tree =
152 list_dependencies_tree(tree, package.clone(), packages, EcoString::new(), invert);
153
154 tree.iter()
155 .try_for_each(|line| writeln!(buffer, "{line}"))
156 .map_err(|e| Error::StandardIo {
157 action: StandardIoAction::Write,
158 err: Some(e.kind()),
159 })
160 } else {
161 writeln!(buffer, "Package not found. Please check the package name.").map_err(|e| {
162 Error::StandardIo {
163 action: StandardIoAction::Write,
164 err: Some(e.kind()),
165 }
166 })
167 }
168}
169
170fn list_dependencies_tree(
171 mut tree: Vec<EcoString>,
172 package: ManifestPackage,
173 packages: Vec<ManifestPackage>,
174 accum: EcoString,
175 invert: bool,
176) -> Vec<EcoString> {
177 let dependencies = packages
178 .iter()
179 .filter(|p| {
180 (invert && p.requirements.contains(&package.name))
181 || (!invert && package.requirements.contains(&p.name))
182 })
183 .cloned()
184 .collect_vec();
185
186 let dependencies = dependencies.iter().sorted().enumerate();
187
188 let deps_length = dependencies.len();
189 for (index, dependency) in dependencies {
190 let is_last = index == deps_length - 1;
191 let prefix = if is_last {
192 UTF8_SYMBOLS.ell
193 } else {
194 UTF8_SYMBOLS.tee
195 };
196
197 tree.push(eco_format!(
198 "{0}{1}{2}{2} {3} v{4}",
199 accum.clone(),
200 prefix,
201 UTF8_SYMBOLS.right,
202 dependency.name,
203 dependency.version
204 ));
205
206 let accum = accum.clone() + (if !is_last { UTF8_SYMBOLS.down } else { " " }) + " ";
207
208 tree = list_dependencies_tree(
209 tree.clone(),
210 dependency.clone(),
211 packages.clone(),
212 accum.clone(),
213 invert,
214 );
215 }
216
217 tree
218}
219
220pub fn outdated(paths: &ProjectPaths) -> Result<()> {
221 let (_, manifest) = get_manifest_details(paths)?;
222
223 let total_packages = manifest
224 .packages
225 .iter()
226 .filter(|package| matches!(package.source, ManifestPackageSource::Hex { .. }))
227 .count();
228
229 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
230 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
231
232 let version_updates = dependency::check_for_version_updates(&manifest, &package_fetcher);
233
234 print!(
235 "{}",
236 pretty_print_outdated_versions(total_packages, version_updates)
237 );
238
239 Ok(())
240}
241
242#[derive(Debug, Clone, Copy)]
243pub enum UseManifest {
244 Yes,
245 No,
246}
247
248pub fn update(paths: &ProjectPaths, packages: Vec<String>) -> Result<()> {
249 let use_manifest = if packages.is_empty() {
250 UseManifest::No
251 } else {
252 UseManifest::Yes
253 };
254
255 // Update specific packages
256 _ = resolve_and_download(
257 paths,
258 cli::Reporter::new(),
259 None,
260 packages.into_iter().map(EcoString::from).collect(),
261 DependencyManagerConfig {
262 use_manifest,
263 check_major_versions: CheckMajorVersions::Yes,
264 },
265 )?;
266
267 Ok(())
268}
269
270/// Edit the manifest.toml file in this proejct, removing all extra requirements and packages
271/// that are no longer present in the gleam.toml config.
272pub fn cleanup<Telem: Telemetry>(paths: &ProjectPaths, telemetry: Telem) -> Result<Manifest> {
273 let span = tracing::info_span!("remove_deps");
274 let _enter = span.enter();
275
276 // We do this before acquiring the build lock so that we don't create the
277 // build directory if there is no gleam.toml
278 crate::config::ensure_config_exists(paths)?;
279
280 let lock = BuildLock::new_packages(paths)?;
281 let _guard: Guard = lock.lock(&telemetry)?;
282
283 // Read the project config
284 let config = crate::root_config(paths)?;
285 let old_manifest = read_manifest_from_disc(paths)?;
286 let mut manifest = old_manifest.clone();
287
288 remove_extra_requirements(&config, &mut manifest)?;
289
290 // Remove any packages that are no longer required due to manifest changes
291 let local = LocalPackages::read_from_disc(paths)?;
292 remove_extra_packages(paths, &local, &manifest, &telemetry)?;
293
294 // Record new state of the packages directory
295 tracing::debug!("writing_manifest_toml");
296 write_manifest_to_disc(paths, &manifest)?;
297 LocalPackages::from_manifest(&manifest).write_to_disc(paths)?;
298
299 let changes = PackageChanges::between_manifests(&old_manifest, &manifest);
300 telemetry.resolved_package_versions(&changes);
301
302 // Cleanup build cache of the root package if there are some changes.
303 // Without this, if a removed dependency is still used, teh build will
304 // succeed, resulting in runtime crash due to missing files.
305 if changes.any_changes() {
306 tracing::debug!("cleaning_root_package_build_cache");
307 for mode in Mode::iter() {
308 for target in Target::iter() {
309 let lock = BuildLock::new_target(paths, mode, target)?;
310 let _guard = lock.lock(&telemetry)?;
311
312 let build_directory_path =
313 paths.build_directory_for_package(mode, target, &config.name);
314 if build_directory_path.exists() {
315 fs::delete_directory(build_directory_path.as_ref())?;
316 }
317 }
318 }
319 }
320
321 Ok(manifest)
322}
323
324/// Remove requirements and unneeded packages from manifest that are no longer present in config.
325fn remove_extra_requirements(config: &PackageConfig, manifest: &mut Manifest) -> Result<()> {
326 // "extra requirements" are all packages that are requirements in the manifest, but no longer
327 // part of the gleam.toml config.
328 let is_extra_requirement = |name: &EcoString| {
329 !config.dev_dependencies.contains_key(name) && !config.dependencies.contains_key(name)
330 };
331
332 // If a requirement is also used as a dependency, we do not want to force-unlock it.
333 // If the dependents get deleted as well, this transitive dependency will be dropped.
334 let is_unlockable_requirement = |name: &EcoString| {
335 manifest
336 .packages
337 .iter()
338 .all(|p| !p.requirements.contains(name))
339 };
340
341 let extra_requirements = manifest
342 .requirements
343 .keys()
344 .filter(|&name| is_extra_requirement(name) && is_unlockable_requirement(name))
345 .cloned()
346 .collect::<Vec<_>>();
347
348 manifest
349 .requirements
350 .retain(|name, _| !is_extra_requirement(name));
351
352 // Unlock all packages that we we want to remove - this removes them and all unneeded
353 // dependencies from `locked`.
354 let mut locked = config.locked(Some(manifest))?;
355 unlock_packages(&mut locked, extra_requirements.as_slice(), Some(manifest))?;
356 // Remove all unlocked packages from the manifest - these are truly no longer needed.
357 manifest
358 .packages
359 .retain(|package| locked.contains_key(&package.name));
360
361 Ok(())
362}
363
364pub fn parse_gleam_add_specifier(package: &str) -> Result<(EcoString, Requirement)> {
365 let Some((package, version)) = package.split_once('@') else {
366 // Default to the latest version available.
367 return Ok((
368 package.into(),
369 Requirement::hex(">= 0.0.0").expect("'>= 0.0.0' should be a valid pubgrub range"),
370 ));
371 };
372
373 // Parse the major and minor from the provided semantic version.
374 let parts = version.split('.').collect::<Vec<_>>();
375 let major = match parts.first() {
376 Some(major) => Ok(major),
377 None => Err(Error::InvalidVersionFormat {
378 input: package.to_string(),
379 error: "Failed to parse semantic major version".to_string(),
380 }),
381 }?;
382 let minor = match parts.get(1) {
383 Some(minor) => minor,
384 None => "0",
385 };
386
387 // Using the major version specifier, calculate the maximum
388 // allowable version (i.e., the next major version).
389 let Ok(num) = major.parse::<usize>() else {
390 return Err(Error::InvalidVersionFormat {
391 input: version.to_string(),
392 error: "Failed to parse semantic major version as integer".to_string(),
393 });
394 };
395
396 let max_ver = [&(num + 1).to_string(), "0", "0"].join(".");
397
398 // Pad the provided version specifier with zeros map to a Hex version.
399 let requirement = match parts.len() {
400 1 | 2 => {
401 let min_ver = [major, minor, "0"].join(".");
402 Requirement::hex(&[">=", &min_ver, "and", "<", &max_ver].join(" "))
403 }
404 3 => Requirement::hex(version),
405 n => {
406 return Err(Error::InvalidVersionFormat {
407 input: version.to_string(),
408 error: format!(
409 "Expected up to 3 numbers in version specifier (MAJOR.MINOR.PATCH), found {n}"
410 ),
411 });
412 }
413 }?;
414
415 Ok((package.into(), requirement))
416}
417
418pub fn resolve_and_download<Telem: Telemetry>(
419 paths: &ProjectPaths,
420 telemetry: Telem,
421 new_package: Option<(Vec<(EcoString, Requirement)>, bool)>,
422 packages_to_update: Vec<EcoString>,
423 config: DependencyManagerConfig,
424) -> Result<Manifest> {
425 // Start event loop so we can run async functions to call the Hex API
426 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
427 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
428
429 let dependency_manager = config.into_dependency_manager(
430 runtime.handle().clone(),
431 package_fetcher,
432 telemetry,
433 Mode::Dev,
434 );
435
436 dependency_manager.resolve_and_download_versions(paths, new_package, packages_to_update)
437}
438
439fn format_versions_and_extract_longest_parts(
440 versions: dependency::PackageVersionDiffs,
441) -> Vec<Vec<String>> {
442 versions
443 .iter()
444 .map(|(name, (v1, v2))| vec![name.to_string(), v1.to_string(), v2.to_string()])
445 .sorted()
446 .collect_vec()
447}
448
449fn pretty_print_major_versions_available(versions: dependency::PackageVersionDiffs) -> String {
450 let versions = format_versions_and_extract_longest_parts(versions);
451
452 format!(
453 "\nThe following dependencies have new major versions available:\n\n{}",
454 space_table(&["Package", "Current", "Latest"], &versions)
455 )
456}
457
458fn pretty_print_version_updates(versions: dependency::PackageVersionDiffs) -> EcoString {
459 let versions = format_versions_and_extract_longest_parts(versions);
460 space_table(&["Package", "Current", "Latest"], &versions)
461}
462
463fn pretty_print_outdated_versions(
464 total_packages: usize,
465 versions: dependency::PackageVersionDiffs,
466) -> EcoString {
467 let summary = eco_format!(
468 "{} of {} packages have newer versions available.",
469 versions.len(),
470 total_packages
471 );
472
473 if versions.is_empty() {
474 eco_format!("{}\n", summary)
475 } else {
476 eco_format!("{}\n\n{}", summary, pretty_print_version_updates(versions))
477 }
478}
479
480async fn add_missing_packages<Telem: Telemetry>(
481 paths: &ProjectPaths,
482 fs: Box<ProjectIO>,
483 manifest: &Manifest,
484 local: &LocalPackages,
485 project_name: EcoString,
486 telemetry: &Telem,
487) -> Result<(), Error> {
488 let missing_packages = local.missing_local_packages(manifest, &project_name);
489
490 let mut num_to_download = 0;
491
492 let missing_git_packages = missing_packages
493 .iter()
494 .copied()
495 .filter(|package| package.is_git())
496 .inspect(|_| {
497 num_to_download += 1;
498 })
499 .collect_vec();
500
501 let mut missing_hex_packages = missing_packages
502 .iter()
503 .copied()
504 .filter(|package| package.is_hex())
505 .inspect(|_| {
506 num_to_download += 1;
507 })
508 .peekable();
509
510 // If we need to download at-least one package
511 if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() {
512 let http = HttpClient::boxed();
513 let downloader = hex::Downloader::new(fs.clone(), fs, http, Untar::boxed(), paths.clone());
514 let start = Instant::now();
515 telemetry.downloading_package("packages");
516 downloader
517 .download_hex_packages(missing_hex_packages, &project_name)
518 .await?;
519 for package in missing_git_packages {
520 let ManifestPackageSource::Git { repo, commit, path } = &package.source else {
521 continue;
522 };
523
524 let checkout =
525 download_git_package(&package.name, repo, commit, path.as_deref(), paths)?;
526 checkout.cleanup()?;
527 }
528 telemetry.packages_downloaded(start, num_to_download);
529 }
530
531 Ok(())
532}
533
534fn remove_extra_packages<Telem: Telemetry>(
535 paths: &ProjectPaths,
536 local: &LocalPackages,
537 manifest: &Manifest,
538 telemetry: &Telem,
539) -> Result<()> {
540 let _guard = BuildLock::lock_all_build(paths, telemetry)?;
541
542 for (package_name, version) in local.extra_local_packages(manifest) {
543 // TODO: test
544 // Delete the package source
545 let path = paths.build_packages_package(&package_name);
546 if path.exists() {
547 tracing::debug!(package=%package_name, version=%version, "removing_unneeded_package");
548 fs::delete_directory(&path)?;
549 }
550
551 // TODO: test
552 // Delete any build artefacts for the package
553 for mode in Mode::iter() {
554 for target in Target::iter() {
555 let name = manifest
556 .packages
557 .iter()
558 .find(|p| p.name == package_name)
559 .map(|p| p.application_name().as_str())
560 .unwrap_or(package_name.as_str());
561 let path = paths.build_directory_for_package(mode, target, name);
562 if path.exists() {
563 tracing::debug!(package=%package_name, version=%version, "deleting_build_cache");
564 fs::delete_directory(&path)?;
565 }
566 }
567 }
568 }
569
570 remove_unused_git_clones(paths, manifest)?;
571 Ok(())
572}
573
574fn remove_unused_git_clones(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
575 let git_directory = paths.build_git_directory();
576 if !git_directory.is_dir() {
577 return Ok(());
578 }
579
580 let expected: HashSet<String> = manifest
581 .packages
582 .iter()
583 .filter_map(|package| match &package.source {
584 ManifestPackageSource::Git {
585 repo,
586 path: Some(_),
587 ..
588 } => Some(git_repo_dir_name(repo)),
589 _ => None,
590 })
591 .collect();
592
593 for entry in fs::read_dir(&git_directory)?.filter_map(Result::ok) {
594 if !expected.contains(entry.file_name()) {
595 tracing::debug!(path=%entry.path(), "removing_unused_git_clone");
596 fs::delete_directory(entry.path())?;
597 }
598 }
599 Ok(())
600}
601
602fn read_manifest_from_disc(paths: &ProjectPaths) -> Result<Manifest> {
603 tracing::debug!("reading_manifest_toml");
604 let manifest_path = paths.manifest();
605 let toml = fs::read(&manifest_path)?;
606 let manifest = toml::from_str(&toml).map_err(|e| Error::FileIo {
607 action: FileIoAction::Parse,
608 kind: FileKind::File,
609 path: manifest_path.clone(),
610 err: Some(e.to_string()),
611 })?;
612 Ok(manifest)
613}
614
615fn write_manifest_to_disc(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
616 let path = paths.manifest();
617 fs::write(&path, &manifest.to_toml(paths.root()))
618}
619
620// This is the container for locally pinned packages, representing the current contents of
621// the `project/build/packages` directory.
622// For descriptions of packages provided by paths and git deps, see the ProvidedPackage struct.
623// The same package may appear in both at different times.
624#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
625struct LocalPackages {
626 #[serde(deserialize_with = "gleam_core::config::map_with_package_name_keys::deserialize")]
627 packages: HashMap<EcoString, Version>,
628 // Git packages can resolve to a new commit (or a new sub-path) without
629 // their version changing, so their on-disc freshness is keyed by commit
630 // and path rather than version alone. Absent for hex and local packages.
631 #[serde(default)]
632 git: HashMap<EcoString, GitState>,
633}
634
635#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
636struct GitState {
637 commit: EcoString,
638 #[serde(default)]
639 path: Option<Utf8PathBuf>,
640}
641
642impl LocalPackages {
643 pub fn extra_local_packages(&self, manifest: &Manifest) -> Vec<(EcoString, Version)> {
644 let manifest_packages: HashSet<_> = manifest
645 .packages
646 .iter()
647 .map(|p| (&p.name, &p.version))
648 .collect();
649 self.packages
650 .iter()
651 .filter(|(n, v)| !manifest_packages.contains(&(&EcoString::from(*n), v)))
652 .map(|(n, v)| (n.clone(), v.clone()))
653 .collect()
654 }
655
656 pub fn missing_local_packages<'a>(
657 &self,
658 manifest: &'a Manifest,
659 root: &str,
660 ) -> Vec<&'a ManifestPackage> {
661 manifest
662 .packages
663 .iter()
664 // We don't need to download the root package
665 .filter(|p| p.name != root)
666 // We don't need to download local packages because we use the linked source directly
667 .filter(|p| !p.is_local())
668 // We don't need to download packages which we already have on disc. For
669 // git packages this means the same commit and path.
670 .filter(|p| !self.has_fresh(p))
671 .collect()
672 }
673
674 fn has_fresh(&self, package: &ManifestPackage) -> bool {
675 match &package.source {
676 ManifestPackageSource::Git { commit, path, .. } => match self.git.get(&package.name) {
677 Some(state) => &state.commit == commit && &state.path == path,
678 None => false,
679 },
680 ManifestPackageSource::Hex { .. } | ManifestPackageSource::Local { .. } => {
681 self.packages.get(package.name.as_str()) == Some(&package.version)
682 }
683 }
684 }
685
686 pub fn read_from_disc(paths: &ProjectPaths) -> Result<Self> {
687 let path = paths.build_packages_toml();
688 if !path.exists() {
689 return Ok(Self {
690 packages: HashMap::new(),
691 git: HashMap::new(),
692 });
693 }
694 let toml = fs::read(&path)?;
695 toml::from_str(&toml).map_err(|e| Error::FileIo {
696 action: FileIoAction::Parse,
697 kind: FileKind::File,
698 path: path.clone(),
699 err: Some(e.to_string()),
700 })
701 }
702
703 pub fn write_to_disc(&self, paths: &ProjectPaths) -> Result<()> {
704 let path = paths.build_packages_toml();
705 let toml = toml::to_string(&self).expect("packages.toml serialization");
706 fs::write(&path, &toml)
707 }
708
709 pub fn from_manifest(manifest: &Manifest) -> Self {
710 let packages = manifest
711 .packages
712 .iter()
713 .map(|p| (p.name.clone(), p.version.clone()))
714 .collect();
715 let git = manifest
716 .packages
717 .iter()
718 .filter_map(|p| match &p.source {
719 ManifestPackageSource::Git { commit, path, .. } => Some((
720 p.name.clone(),
721 GitState {
722 commit: commit.clone(),
723 path: path.clone(),
724 },
725 )),
726 ManifestPackageSource::Hex { .. } | ManifestPackageSource::Local { .. } => None,
727 })
728 .collect();
729
730 Self { packages, git }
731 }
732}
733
734#[test]
735fn local_packages_deserialise_ok() {
736 let toml = r#"
737[packages]
738gleam_stdlib = "1.0.0"
739gleam_otp = "1.1.0"
740"#;
741 let packages: LocalPackages = toml::from_str(toml).unwrap();
742 assert_eq!(
743 packages,
744 LocalPackages {
745 packages: HashMap::from_iter([
746 ("gleam_stdlib".into(), Version::new(1, 0, 0)),
747 ("gleam_otp".into(), Version::new(1, 1, 0)),
748 ]),
749 git: HashMap::new(),
750 }
751 )
752}
753
754#[test]
755fn local_packages_deserialise_invalid_name() {
756 let toml = r#"
757[packages]
758gleam_stdlib = "1.0.0"
759"../../stuff" = "1.1.0"
760"#;
761 let error = toml::from_str::<LocalPackages>(toml)
762 .expect_err("should fail to deserialise because of invalid name");
763 insta::assert_snapshot!(insta::internals::AutoName, error.to_string());
764}
765
766fn is_same_requirements(
767 requirements1: &HashMap<EcoString, Requirement>,
768 requirements2: &HashMap<EcoString, Requirement>,
769 root_path: &Utf8Path,
770) -> Result<bool> {
771 if requirements1.len() != requirements2.len() {
772 return Ok(false);
773 }
774
775 for (key, requirement1) in requirements1 {
776 if !same_requirements(requirement1, requirements2.get(key), root_path)? {
777 return Ok(false);
778 }
779 }
780
781 Ok(true)
782}
783
784/// Returns true if all path dependency configs are unchanged since last build.
785///
786/// If any of the path dependency configs have changed that means that we need to
787/// re-perform dependency resolution, as their dependencies could have changed
788/// themselves.
789///
790/// We use gleam.toml rather than manifest.toml as:
791///
792/// 1. The dependency requirements could have changed but resolution not have been
793/// run in that package yet, so the manifest would be the same, resulting in us
794/// failing to detect that resolution is required.
795///
796/// 2. Dependency manifests are not used in any way, so a change in the manifest
797/// may not have any impact on this package.
798///
799/// This does mean that changes unrelated to the path dependency's dependencies
800/// will trigger resolution, but gleam.toml is edited rarely, and no-change
801/// resolution is fast enough, so that's OK.
802///
803/// Note: This does not check path dependencies of path dependencies! Changes to
804/// their configs will fail to be picked up. To resolve this we would need to keep
805/// a list of all the path dependencies in the project, instead of only the direct
806/// path dependencies.
807///
808fn path_dependency_configs_unchanged(
809 requirements: &HashMap<EcoString, Requirement>,
810 paths: &ProjectPaths,
811) -> Result<bool> {
812 for (name, requirement) in requirements {
813 let Requirement::Path { path } = requirement else {
814 continue;
815 };
816
817 let config_path = paths.path_dependency_gleam_toml_path(path);
818 let fingerprint_path = paths.dependency_gleam_toml_fingerprint_path(name.as_str());
819
820 // Check mtimes before hashing, to avoid extra work
821 if fingerprint_path.exists() {
822 let config_time = fs::modification_time(&config_path)?;
823 let fingerprint_time = fs::modification_time(&fingerprint_path)?;
824 if config_time <= fingerprint_time {
825 continue;
826 }
827 };
828
829 let config_text = fs::read(&config_path)?;
830 let current_fingerprint = SourceFingerprint::new(&config_text).to_numerical_string();
831
832 // If cached hash file doesn't exist, this is the first time we're checking this dependency
833 if !fingerprint_path.exists() {
834 // Save the current hash for future comparisons
835 fs::write(&fingerprint_path, ¤t_fingerprint)?;
836 return Ok(false);
837 }
838
839 let previous_fingerprint = fs::read(&fingerprint_path)?;
840
841 if previous_fingerprint != current_fingerprint {
842 tracing::debug!("path_dependency_config_changed_forcing_rebuild");
843 fs::write(&fingerprint_path, ¤t_fingerprint)?;
844 return Ok(false);
845 }
846 }
847
848 Ok(true)
849}
850
851fn same_requirements(
852 requirement1: &Requirement,
853 requirement2: Option<&Requirement>,
854 root_path: &Utf8Path,
855) -> Result<bool> {
856 let (left, right) = match (requirement1, requirement2) {
857 (Requirement::Path { path: path1 }, Some(Requirement::Path { path: path2 })) => {
858 let left = fs::canonicalise(&root_path.join(path1))?;
859 let right = fs::canonicalise(&root_path.join(path2))?;
860 (left, right)
861 }
862 (_, Some(requirement2)) => return Ok(requirement1 == requirement2),
863 (_, None) => return Ok(false),
864 };
865
866 Ok(left == right)
867}
868
869#[derive(Clone, Eq, PartialEq, Debug)]
870struct ProvidedPackage {
871 version: Version,
872 source: ProvidedPackageSource,
873 requirements: HashMap<EcoString, hexpm::version::Range>,
874}
875
876#[derive(Clone, Eq, Debug)]
877enum ProvidedPackageSource {
878 Git {
879 repo: EcoString,
880 commit: EcoString,
881 path: Option<Utf8PathBuf>,
882 },
883 Local {
884 path: Utf8PathBuf,
885 },
886}
887
888impl ProvidedPackage {
889 fn to_hex_package(&self, name: &EcoString) -> hexpm::Package {
890 let requirements = self
891 .requirements
892 .iter()
893 .map(|(name, version)| {
894 (
895 name.as_str().into(),
896 hexpm::Dependency {
897 requirement: version.clone(),
898 optional: false,
899 app: None,
900 repository: None,
901 },
902 )
903 })
904 .collect();
905 let release = hexpm::Release {
906 version: self.version.clone(),
907 requirements,
908 retirement_status: None,
909 outer_checksum: vec![],
910 meta: (),
911 };
912 hexpm::Package {
913 name: name.as_str().into(),
914 repository: "local".into(),
915 releases: vec![release],
916 }
917 }
918
919 fn to_manifest_package(&self, name: &str) -> ManifestPackage {
920 let mut package = ManifestPackage {
921 name: name.into(),
922 version: self.version.clone(),
923 otp_app: None, // Note, this will probably need to be set to something eventually
924 build_tools: vec!["gleam".into()],
925 requirements: self.requirements.keys().cloned().collect(),
926 source: self.source.to_manifest_package_source(),
927 };
928 package.requirements.sort();
929 package
930 }
931}
932
933impl ProvidedPackageSource {
934 fn to_manifest_package_source(&self) -> ManifestPackageSource {
935 match self {
936 Self::Git { repo, commit, path } => ManifestPackageSource::Git {
937 repo: repo.clone(),
938 commit: commit.clone(),
939 path: path.clone(),
940 },
941 Self::Local { path } => ManifestPackageSource::Local { path: path.clone() },
942 }
943 }
944
945 fn to_toml(&self) -> String {
946 match self {
947 Self::Git { repo, commit, path } => match path {
948 Some(path) => {
949 format!(r#"{{ repo: "{repo}", commit: "{commit}", path: "{path}" }}"#)
950 }
951 None => format!(r#"{{ repo: "{repo}", commit: "{commit}" }}"#),
952 },
953 Self::Local { path } => {
954 format!(r#"{{ path: "{path}" }}"#)
955 }
956 }
957 }
958}
959
960impl PartialEq for ProvidedPackageSource {
961 fn eq(&self, other: &Self) -> bool {
962 match (self, other) {
963 (Self::Local { path: own_path }, Self::Local { path: other_path }) => {
964 is_same_file(own_path, other_path).unwrap_or(false)
965 }
966
967 (
968 Self::Git {
969 repo: own_repo,
970 commit: own_commit,
971 path: own_path,
972 },
973 Self::Git {
974 repo: other_repo,
975 commit: other_commit,
976 path: other_path,
977 },
978 ) => own_repo == other_repo && own_commit == other_commit && own_path == other_path,
979
980 (Self::Git { .. }, Self::Local { .. }) | (Self::Local { .. }, Self::Git { .. }) => {
981 false
982 }
983 }
984 }
985}
986
987/// Where a provided package came from. Git sources carry the on-disc repository
988/// root so path dependencies can be resolved relative to it.
989enum SourceContext<'a> {
990 Local {
991 path: Utf8PathBuf,
992 },
993 Git {
994 repo: EcoString,
995 commit: EcoString,
996 path: Option<Utf8PathBuf>,
997 repo_root: &'a Utf8Path,
998 },
999}
1000
1001impl SourceContext<'_> {
1002 fn to_provided_source(&self) -> ProvidedPackageSource {
1003 match self {
1004 Self::Local { path } => ProvidedPackageSource::Local { path: path.clone() },
1005 Self::Git {
1006 repo, commit, path, ..
1007 } => ProvidedPackageSource::Git {
1008 repo: repo.clone(),
1009 commit: commit.clone(),
1010 path: path.clone(),
1011 },
1012 }
1013 }
1014}
1015
1016/// Provide a package from a local project
1017fn provide_local_package(
1018 package_name: EcoString,
1019 package_path: &Utf8Path,
1020 parent_path: &Utf8Path,
1021 project_paths: &ProjectPaths,
1022 provided: &mut HashMap<EcoString, ProvidedPackage>,
1023 parents: &mut Vec<EcoString>,
1024) -> Result<hexpm::version::Range> {
1025 let package_path = if package_path.is_absolute() {
1026 package_path.to_path_buf()
1027 } else {
1028 fs::canonicalise(&parent_path.join(package_path))?
1029 };
1030
1031 provide_package(
1032 package_name,
1033 package_path.clone(),
1034 SourceContext::Local { path: package_path },
1035 project_paths,
1036 provided,
1037 parents,
1038 )
1039}
1040
1041/// Resolve a path dependency of a git package to its canonical filesystem
1042/// location and its repository-relative path.
1043fn resolve_git_path_package(
1044 package_name: &EcoString,
1045 path: &Utf8Path,
1046 repo: &EcoString,
1047 parent_path: &Utf8Path,
1048 repo_root: &Utf8Path,
1049) -> Result<(Utf8PathBuf, Utf8PathBuf)> {
1050 let location = parent_path.join(path);
1051 if !location.is_dir() {
1052 return Err(Error::GitDependencyPathNotFound {
1053 package: package_name.to_string(),
1054 path: path.to_string(),
1055 repo: repo.to_string(),
1056 });
1057 }
1058
1059 // The path may name a symlink that points outside the repository
1060 // checkout, so resolve it and take the repository-relative path from the
1061 // canonical location.
1062 let package_path = fs::canonicalise(&location)?;
1063 let repo_path = package_path
1064 .strip_prefix(repo_root)
1065 .map_err(|_| Error::GitDependencyPathNotFound {
1066 package: package_name.to_string(),
1067 path: path.to_string(),
1068 repo: repo.to_string(),
1069 })?
1070 .to_path_buf();
1071
1072 Ok((package_path, repo_path))
1073}
1074
1075fn execute_command(command: &mut Command) -> Result<std::process::Output> {
1076 let result = command.output();
1077 match result {
1078 Ok(output) if output.status.success() => Ok(output),
1079 Ok(output) => {
1080 let reason = match String::from_utf8(output.stderr) {
1081 Ok(stderr) => ShellCommandFailureReason::ShellCommandError(stderr),
1082 Err(_) => ShellCommandFailureReason::Unknown,
1083 };
1084 Err(Error::ShellCommand {
1085 program: "git".into(),
1086 reason,
1087 })
1088 }
1089 Err(error) => Err(git_io_error(error)),
1090 }
1091}
1092
1093/// A `git` command with its working directory set to `dir`.
1094fn git_command(dir: &Utf8Path) -> Command {
1095 let mut command = Command::new("git");
1096 let _ = command.current_dir(dir);
1097 command
1098}
1099
1100/// Map an IO error from spawning `git` onto the appropriate `Error`.
1101fn git_io_error(error: std::io::Error) -> Error {
1102 match error.kind() {
1103 ErrorKind::NotFound => Error::ShellProgramNotFound {
1104 program: "git".into(),
1105 os: fs::get_os(),
1106 },
1107 other => Error::ShellCommand {
1108 program: "git".into(),
1109 reason: ShellCommandFailureReason::IoError(other),
1110 },
1111 }
1112}
1113
1114/// Parse the trimmed UTF-8 stdout of a `git` command.
1115fn git_stdout(output: std::process::Output) -> EcoString {
1116 String::from_utf8(output.stdout)
1117 .expect("Output should be UTF-8")
1118 .trim()
1119 .into()
1120}
1121
1122fn git_repo_dir_name(repo: &str) -> String {
1123 let name = repo
1124 .trim_end_matches('/')
1125 .trim_end_matches(".git")
1126 .rsplit(['/', ':', '\\'])
1127 .next()
1128 .filter(|name| !name.is_empty())
1129 .unwrap_or("repo");
1130 let hash = xxhash_rust::xxh3::xxh3_64(repo.as_bytes());
1131 format!("{name}-{hash:016x}")
1132}
1133
1134fn git_staging_path(project_paths: &ProjectPaths, repo: &str, package_name: &str) -> Utf8PathBuf {
1135 project_paths.build_git_repo(&format!(
1136 "{}-{package_name}-staging",
1137 git_repo_dir_name(repo)
1138 ))
1139}
1140
1141enum GitCheckout {
1142 InPlace {
1143 commit: EcoString,
1144 },
1145 Staged {
1146 commit: EcoString,
1147 staging_path: Utf8PathBuf,
1148 },
1149}
1150
1151impl GitCheckout {
1152 fn cleanup(self) -> Result<()> {
1153 // The clones dangling worktree registration is left for the next
1154 // download to prune before it re-adds the staging worktree.
1155 if let Self::Staged { staging_path, .. } = self {
1156 fs::delete_directory(&staging_path)?;
1157 }
1158 Ok(())
1159 }
1160}
1161
1162/// Downloads a git package from a remote repository.
1163///
1164/// For a package at the root of its repository the commands that are run look
1165/// like this, cloning directly into `build/packages/<name>`:
1166///
1167/// ```sh
1168/// git init
1169/// git remote remove origin
1170/// git remote add origin <repo>
1171/// git fetch origin
1172/// git checkout <ref>
1173/// git rev-parse HEAD
1174/// ```
1175///
1176/// For a package in a subdirectory of its repository the repository is
1177/// instead cloned into `build/git/<repo-name>-<hash>` as a bare repository
1178/// with no work tree, and the package is checked out into a transient staging
1179/// worktree which the subdirectory is hard-linked out of:
1180///
1181/// ```sh
1182/// git init --bare
1183/// git remote remove origin
1184/// git remote add origin <repo>
1185/// git fetch origin
1186/// git rev-parse --verify --quiet <ref>^{commit}
1187/// git worktree prune
1188/// git worktree add --force --detach <clone>-<pkg>-staging <commit>
1189/// ```
1190///
1191/// The staging worktree exists only for the duration of one package download.
1192/// It is deleted (and the clone's worktree registration pruned) by
1193/// `GitCheckout::cleanup` once the package has been hard-linked and read.
1194///
1195/// This is somewhat inefficient as we have to fetch the entire git history before
1196/// switching to the exact commit we want. There a few alternatives to this:
1197///
1198/// - `git clone --depth 1 --branch="<ref>"` This works, but only allows us to use
1199/// branch names as refs, however we want to allow commit hashes as well.
1200/// - `git fetch --depth 1 origin <ref>` Similarly, this imposes an unwanted
1201/// restriction. `git fetch` only allows branch names or full commit hashes,
1202/// but we want to allow partial hashes as well.
1203///
1204/// Since Git dependencies will be used quite rarely, this option was settled upon
1205/// because it allows branch names, full and partial commit hashes as refs.
1206///
1207/// In the future we can optimise this more, for example first checking if we
1208/// are already checked out to the commit stored in the manifest, or by only
1209/// fetching the history without the objects to resolve partial commit hashes.
1210/// For now though this is good enough until it become an actual performance
1211/// problem.
1212///
1213fn download_git_package(
1214 package_name: &str,
1215 repo: &str,
1216 ref_: &str,
1217 path: Option<&Utf8Path>,
1218 project_paths: &ProjectPaths,
1219) -> Result<GitCheckout> {
1220 match path {
1221 None => download_git_package_in_place(package_name, repo, ref_, project_paths),
1222 Some(subdir) => {
1223 download_git_package_to_staged_path(package_name, repo, ref_, project_paths, subdir)
1224 }
1225 }
1226}
1227
1228fn download_git_package_in_place(
1229 package_name: &str,
1230 repo: &str,
1231 ref_: &str,
1232 project_paths: &ProjectPaths,
1233) -> Result<GitCheckout, Error> {
1234 let clone_path = project_paths.build_packages_package(package_name);
1235 prepare_git_clone(&clone_path, repo, CloneFormat::WorkTree)?;
1236 let _ = execute_command(git_command(&clone_path).arg("checkout").arg(ref_))?;
1237 let output = execute_command(git_command(&clone_path).arg("rev-parse").arg("HEAD"))?;
1238 let commit = git_stdout(output);
1239
1240 Ok(GitCheckout::InPlace { commit })
1241}
1242
1243fn download_git_package_to_staged_path(
1244 package_name: &str,
1245 repo: &str,
1246 ref_: &str,
1247 project_paths: &ProjectPaths,
1248 subdir: &Utf8Path,
1249) -> Result<GitCheckout, Error> {
1250 let clone_path = project_paths.build_git_repo(&git_repo_dir_name(repo));
1251 prepare_git_clone(&clone_path, repo, CloneFormat::Bare)?;
1252
1253 let commit = resolve_git_ref(&clone_path, repo, ref_)?;
1254
1255 // Delete any staging worktree left behind by a previous crash, and prune
1256 // its registration from the clone so `git worktree add` can reuse the
1257 // path.
1258 let staging_path = git_staging_path(project_paths, repo, package_name);
1259 fs::delete_directory(&staging_path)?;
1260 let _ = git_command(&clone_path)
1261 .arg("worktree")
1262 .arg("prune")
1263 .output();
1264
1265 let _ = execute_command(
1266 git_command(&clone_path)
1267 .arg("worktree")
1268 .arg("add")
1269 .arg("--force")
1270 .arg("--detach")
1271 .arg(&staging_path)
1272 .arg(commit.as_str()),
1273 )?;
1274
1275 let Some(subdir_source) = resolve_git_subdir(&staging_path, subdir) else {
1276 return Err(Error::GitDependencyPathNotFound {
1277 package: package_name.into(),
1278 path: subdir.to_string(),
1279 repo: repo.into(),
1280 });
1281 };
1282
1283 let package_path = project_paths.build_packages_package(package_name);
1284 fs::delete_directory(&package_path)?;
1285 fs::mkdir(&package_path)?;
1286 fs::hardlink_dir(&subdir_source, &package_path)?;
1287
1288 Ok(GitCheckout::Staged {
1289 commit,
1290 staging_path,
1291 })
1292}
1293
1294#[derive(Clone, Copy)]
1295enum CloneFormat {
1296 /// A regular clone with a checked-out work tree.
1297 WorkTree,
1298 /// A bare repository with no work tree.
1299 Bare,
1300}
1301
1302/// Initialise (or reuse) a git clone at the given path and fetch from the
1303/// remote repository.
1304fn prepare_git_clone(clone_path: &Utf8Path, repo: &str, format: CloneFormat) -> Result<()> {
1305 // If the clone path exists but is not the kind of git repo we expect, we
1306 // need to remove the directory.
1307 let reusable = match format {
1308 CloneFormat::Bare => fs::is_bare_git_repo_root(clone_path),
1309 CloneFormat::WorkTree => fs::is_git_work_tree_root(clone_path),
1310 };
1311
1312 if !reusable {
1313 fs::delete_directory(clone_path)?;
1314 }
1315
1316 fs::mkdir(clone_path)?;
1317
1318 let mut init = git_command(clone_path);
1319 let _ = init.arg("init");
1320 if matches!(format, CloneFormat::Bare) {
1321 let _ = init.arg("--bare");
1322 }
1323 let _ = execute_command(&mut init)?;
1324
1325 // If this directory already exists, but the remote URL has been edited in
1326 // `gleam.toml` without a `gleam clean`, `git remote add` will fail, causing
1327 // the remote to be stuck as the original value. Here we remove the remote
1328 // first, which ensures that `git remote add` properly add the remote each
1329 // time. If this fails, that means we haven't set the remote in the first
1330 // place, so we can safely ignore the error.
1331 let _ = git_command(clone_path)
1332 .arg("remote")
1333 .arg("remove")
1334 .arg("origin")
1335 .output();
1336
1337 let _ = execute_command(
1338 git_command(clone_path)
1339 .arg("remote")
1340 .arg("add")
1341 .arg("origin")
1342 .arg(repo),
1343 )?;
1344
1345 let _ = execute_command(git_command(clone_path).arg("fetch").arg("origin"))?;
1346
1347 Ok(())
1348}
1349
1350/// Resolve a ref (a branch name, tag, or full or partial commit hash) to a
1351/// full commit hash using the objects fetched into the clone, without
1352/// checking anything out. Branch names only exist as remote-tracking refs in
1353/// the never-checked-out clone, so when the ref does not resolve directly we
1354/// fall back to `origin/<ref>`.
1355fn resolve_git_ref(clone_path: &Utf8Path, repo: &str, ref_: &str) -> Result<EcoString> {
1356 let revisions = [
1357 format!("{ref_}^{{commit}}"),
1358 format!("origin/{ref_}^{{commit}}"),
1359 ];
1360
1361 for revision in revisions {
1362 let result = git_command(clone_path)
1363 .arg("rev-parse")
1364 .arg("--verify")
1365 .arg("--quiet")
1366 .arg(&revision)
1367 .output();
1368
1369 match result {
1370 // A failed rev-parse is expected when the ref form doesn't match
1371 // this pattern, so try the next one.
1372 Ok(output) if !output.status.success() => (),
1373 Ok(output) => return Ok(git_stdout(output)),
1374 Err(error) => return Err(git_io_error(error)),
1375 }
1376 }
1377
1378 Err(Error::ShellCommand {
1379 program: "git".into(),
1380 reason: ShellCommandFailureReason::ShellCommandError(format!(
1381 "Unable to resolve git ref `{ref_}` for repository `{repo}`\n"
1382 )),
1383 })
1384}
1385
1386/// Resolves a subdirectory within a cloned git repository, ensuring that the
1387/// resolved directory (after following any symlinks) is still inside the
1388/// repository checkout. Returns `None` if the directory does not exist or
1389/// escapes the checkout.
1390fn resolve_git_subdir(clone_path: &Utf8Path, subdir: &Utf8Path) -> Option<Utf8PathBuf> {
1391 let subdir_source = clone_path.join(subdir);
1392 if !subdir_source.is_dir() {
1393 return None;
1394 }
1395
1396 let canonical_clone = fs::canonicalise(clone_path).ok()?;
1397 let canonical_subdir = fs::canonicalise(&subdir_source).ok()?;
1398 if !canonical_subdir.starts_with(&canonical_clone) {
1399 return None;
1400 }
1401
1402 Some(canonical_subdir)
1403}
1404
1405/// Provide a package from a git repository
1406fn provide_git_package(
1407 package_name: EcoString,
1408 repo: &str,
1409 // A git ref, such as a branch name, commit hash or tag name
1410 ref_: &str,
1411 path: Option<Utf8PathBuf>,
1412 project_paths: &ProjectPaths,
1413 provided: &mut HashMap<EcoString, ProvidedPackage>,
1414 parents: &mut Vec<EcoString>,
1415) -> Result<hexpm::version::Range> {
1416 let checkout = download_git_package(&package_name, repo, ref_, path.as_deref(), project_paths)?;
1417 let (commit, staging_path) = match &checkout {
1418 GitCheckout::InPlace { commit } => (commit, None),
1419 GitCheckout::Staged {
1420 commit,
1421 staging_path,
1422 } => (commit, Some(staging_path)),
1423 };
1424
1425 // Use the package's location in the staging worktree, where its gleam.toml
1426 // lives, not build/packages/. A `../sibling` path dep is resolved next to
1427 // that gleam.toml, and the sibling is only present in the worktree.
1428 let (package_path, repo_root) = match (&path, staging_path) {
1429 (Some(subdir), Some(staging_path)) => {
1430 let repo_root = fs::canonicalise(staging_path)?;
1431 (fs::canonicalise(&staging_path.join(subdir))?, repo_root)
1432 }
1433 _ => {
1434 let package_path =
1435 fs::canonicalise(&project_paths.build_packages_package(&package_name))?;
1436 (package_path.clone(), package_path)
1437 }
1438 };
1439
1440 let version = provide_package(
1441 package_name,
1442 package_path,
1443 SourceContext::Git {
1444 repo: repo.into(),
1445 commit: commit.clone(),
1446 path,
1447 repo_root: &repo_root,
1448 },
1449 project_paths,
1450 provided,
1451 parents,
1452 )?;
1453
1454 checkout.cleanup()?;
1455 Ok(version)
1456}
1457
1458/// Adds a gleam project located at a specific path to the list of "provided packages"
1459fn provide_package(
1460 package_name: EcoString,
1461 package_path: Utf8PathBuf,
1462 source: SourceContext<'_>,
1463 project_paths: &ProjectPaths,
1464 provided: &mut HashMap<EcoString, ProvidedPackage>,
1465 parents: &mut Vec<EcoString>,
1466) -> Result<hexpm::version::Range> {
1467 let package_source = source.to_provided_source();
1468
1469 // Return early if a package cycle is detected
1470 if parents.contains(&package_name) {
1471 let mut last_cycle = parents
1472 .split(|p| p == &package_name)
1473 .next_back()
1474 .unwrap_or_default()
1475 .to_vec();
1476 last_cycle.push(package_name);
1477 return Err(Error::PackageCycle {
1478 packages: last_cycle,
1479 });
1480 }
1481 // Check that we do not have a cached version of this package already
1482 match provided.get(&package_name) {
1483 Some(package) if package.source == package_source => {
1484 // This package has already been provided from this source, return the version
1485 let version = hexpm::version::Range::new(format!("== {}", package.version))
1486 .expect("== {version} should be a valid range");
1487 return Ok(version);
1488 }
1489 Some(package) => {
1490 // This package has already been provided from a different source which conflicts
1491 return Err(Error::ProvidedDependencyConflict {
1492 package: package_name.into(),
1493 source_1: package_source.to_toml(),
1494 source_2: package.source.to_toml(),
1495 });
1496 }
1497 None => (),
1498 }
1499 // Load the package
1500 let config = crate::config::read(package_path.join("gleam.toml"))?;
1501 // Check that we are loading the correct project
1502 if config.name != package_name {
1503 return Err(Error::WrongDependencyProvided {
1504 expected: package_name.into(),
1505 path: package_path.to_path_buf(),
1506 found: config.name.into(),
1507 });
1508 };
1509 // Walk the requirements of the package
1510 let mut requirements = HashMap::new();
1511 parents.push(package_name);
1512 for (name, requirement) in config.dependencies.into_iter() {
1513 let version = match requirement {
1514 Requirement::Hex { version } => version,
1515 Requirement::Path { path } => match &source {
1516 // A path dependency of a git package points to another
1517 // package within the same repository, so lock it as a git
1518 // source to keep the manifest portable.
1519 SourceContext::Git {
1520 repo,
1521 commit,
1522 repo_root,
1523 ..
1524 } => {
1525 let (child_path, child_repo_path) =
1526 resolve_git_path_package(&name, &path, repo, &package_path, repo_root)?;
1527 // A path dependency resolving to the repository root has an
1528 // empty repo-relative path. Normalise it to `None` so it
1529 // matches a package declared directly as a git dependency
1530 // with no path.
1531 let child_repo_path = if child_repo_path.as_str().is_empty() {
1532 None
1533 } else {
1534 Some(child_repo_path)
1535 };
1536
1537 provide_package(
1538 name.clone(),
1539 child_path,
1540 SourceContext::Git {
1541 repo: repo.clone(),
1542 commit: commit.clone(),
1543 path: child_repo_path,
1544 repo_root,
1545 },
1546 project_paths,
1547 provided,
1548 parents,
1549 )?
1550 }
1551 SourceContext::Local { .. } => {
1552 // Recursively walk local packages
1553 provide_local_package(
1554 name.clone(),
1555 &path,
1556 &package_path,
1557 project_paths,
1558 provided,
1559 parents,
1560 )?
1561 }
1562 },
1563 Requirement::Git { git, ref_, path } => provide_git_package(
1564 name.clone(),
1565 &git,
1566 &ref_,
1567 path,
1568 project_paths,
1569 provided,
1570 parents,
1571 )?,
1572 };
1573 let _ = requirements.insert(name, version);
1574 }
1575 let _ = parents.pop();
1576 // Add the package to the provided packages dictionary
1577 let version = hexpm::version::Range::new(format!("== {}", config.version))
1578 .expect("== {version} should be a valid range");
1579 let _ = provided.insert(
1580 config.name,
1581 ProvidedPackage {
1582 version: config.version,
1583 source: package_source,
1584 requirements,
1585 },
1586 );
1587 // Return the version
1588 Ok(version)
1589}
1590
1591/// Unlocks specified packages and their unique dependencies.
1592///
1593/// If a manifest is provided, it also unlocks indirect dependencies that are
1594/// not required by any other package or the root project.
1595pub fn unlock_packages(
1596 locked: &mut HashMap<EcoString, Version>,
1597 packages_to_unlock: &[EcoString],
1598 manifest: Option<&Manifest>,
1599) -> Result<()> {
1600 if let Some(manifest) = manifest {
1601 let mut packages_to_unlock: Vec<EcoString> = packages_to_unlock.to_vec();
1602
1603 while let Some(package_name) = packages_to_unlock.pop() {
1604 if locked.remove(&package_name).is_some()
1605 && let Some(package) = manifest.packages.iter().find(|p| p.name == package_name)
1606 {
1607 let deps_to_unlock = find_deps_to_unlock(package, locked, manifest);
1608 packages_to_unlock.extend(deps_to_unlock);
1609 }
1610 }
1611 } else {
1612 for package_name in packages_to_unlock {
1613 let _ = locked.remove(package_name);
1614 }
1615 }
1616
1617 Ok(())
1618}
1619
1620/// Identifies which dependencies of a package should be unlocked.
1621///
1622/// A dependency is eligible for unlocking if it is currently locked,
1623/// is not a root dependency, and is not required by any locked package.
1624fn find_deps_to_unlock(
1625 package: &ManifestPackage,
1626 locked: &HashMap<EcoString, Version>,
1627 manifest: &Manifest,
1628) -> Vec<EcoString> {
1629 package
1630 .requirements
1631 .iter()
1632 .filter(|&dep| {
1633 locked.contains_key(dep)
1634 && !manifest.requirements.contains_key(dep)
1635 && manifest
1636 .packages
1637 .iter()
1638 .all(|p| !locked.contains_key(&p.name) || !p.requirements.contains(dep))
1639 })
1640 .cloned()
1641 .collect()
1642}
1643
1644/// Determine the information to add to the manifest for a specific package
1645async fn lookup_package(
1646 name: String,
1647 version: Version,
1648 provided: &HashMap<EcoString, ProvidedPackage>,
1649) -> Result<ManifestPackage> {
1650 match provided.get(name.as_str()) {
1651 Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
1652 None => {
1653 let config = hexpm::Config::new();
1654 let release =
1655 hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?;
1656 let build_tools = release
1657 .meta
1658 .build_tools
1659 .iter()
1660 .map(|s| EcoString::from(s.as_str()))
1661 .collect_vec();
1662 let requirements = release
1663 .requirements
1664 .keys()
1665 .map(|s| EcoString::from(s.as_str()))
1666 .collect_vec();
1667 Ok(ManifestPackage {
1668 name: name.into(),
1669 version,
1670 otp_app: Some(release.meta.app.into()),
1671 build_tools,
1672 requirements,
1673 source: ManifestPackageSource::Hex {
1674 outer_checksum: Base16Checksum(release.outer_checksum),
1675 },
1676 })
1677 }
1678 }
1679}
1680
1681struct PackageFetcher {
1682 runtime_cache: RefCell<HashMap<String, Rc<hexpm::Package>>>,
1683 runtime: tokio::runtime::Handle,
1684 http: HttpClient,
1685}
1686
1687impl PackageFetcher {
1688 pub fn new(runtime: tokio::runtime::Handle) -> Self {
1689 Self {
1690 runtime_cache: RefCell::new(HashMap::new()),
1691 runtime,
1692 http: HttpClient::new(),
1693 }
1694 }
1695
1696 /// Caches the result of `get_dependencies` so that we don't need to make a network request.
1697 /// Currently dependencies are fetched during initial version resolution, and then during check
1698 /// for major version availability.
1699 fn cache_package(&self, package: &str, result: Rc<hexpm::Package>) {
1700 let mut runtime_cache = self.runtime_cache.borrow_mut();
1701 let _ = runtime_cache.insert(package.to_string(), result);
1702 }
1703}
1704
1705#[derive(Debug)]
1706pub struct Untar;
1707
1708impl Untar {
1709 pub fn boxed() -> Box<Self> {
1710 Box::new(Self)
1711 }
1712}
1713
1714impl TarUnpacker for Untar {
1715 fn io_result_entries<'a>(
1716 &self,
1717 archive: &'a mut tar::Archive<WrappedReader>,
1718 ) -> std::io::Result<tar::Entries<'a, WrappedReader>> {
1719 archive.entries()
1720 }
1721
1722 fn io_result_unpack(
1723 &self,
1724 path: &Utf8Path,
1725 mut archive: tar::Archive<GzDecoder<tar::Entry<'_, WrappedReader>>>,
1726 ) -> std::io::Result<()> {
1727 archive.unpack(path)
1728 }
1729}
1730
1731impl dependency::PackageFetcher for PackageFetcher {
1732 fn get_dependencies(&self, package: &str) -> Result<Rc<hexpm::Package>, PackageFetchError> {
1733 {
1734 let runtime_cache = self.runtime_cache.borrow();
1735 let result = runtime_cache.get(package);
1736
1737 if let Some(result) = result {
1738 return Ok(result.clone());
1739 }
1740 }
1741
1742 tracing::debug!(package = package, "looking_up_hex_package");
1743 let config = hexpm::Config::new();
1744 let request = hexpm::repository_v2_get_package_request(package, None, &config);
1745 let response = self
1746 .runtime
1747 .block_on(self.http.send(request))
1748 .map_err(PackageFetchError::fetch_error)?;
1749
1750 let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
1751 .map_err(|e| PackageFetchError::from_api_error(e, package))?;
1752 let pkg = Rc::new(pkg);
1753 let pkg_ref = Rc::clone(&pkg);
1754 self.cache_package(package, pkg);
1755 Ok(pkg_ref)
1756 }
1757}