nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
59 kB
1769 lines
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2021 The Gleam contributors
3
4mod dependency_manager;
5
6use std::{
7 cell::RefCell,
8 collections::{HashMap, HashSet},
9 io::ErrorKind,
10 process::Command,
11 rc::Rc,
12 time::Instant,
13};
14
15use camino::{Utf8Path, Utf8PathBuf};
16use ecow::{EcoString, eco_format};
17use flate2::read::GzDecoder;
18use gleam_core::{
19 Error, Result,
20 build::{Mode, SourceFingerprint, Target, Telemetry},
21 config::PackageConfig,
22 dependency::{self, PackageFetchError},
23 error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
24 hex::{self, HEXPM_PUBLIC_KEY},
25 io::{HttpClient as _, TarUnpacker, WrappedReader},
26 manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, PackageChanges},
27 paths::ProjectPaths,
28 requirement::Requirement,
29};
30use hexpm::version::Version;
31use itertools::Itertools;
32use same_file::is_same_file;
33use strum::IntoEnumIterator;
34
35pub use dependency_manager::DependencyManagerConfig;
36
37#[cfg(test)]
38mod tests;
39
40use crate::{
41 TreeOptions,
42 build_lock::{BuildLock, Guard},
43 cli,
44 fs::{self, ProjectIO},
45 http::HttpClient,
46 text_layout::space_table,
47};
48
49struct Symbols {
50 down: &'static str,
51 tee: &'static str,
52 ell: &'static str,
53 right: &'static str,
54}
55
56static UTF8_SYMBOLS: Symbols = Symbols {
57 down: "│",
58 tee: "├",
59 ell: "└",
60 right: "─",
61};
62
63/// When set to `Yes`, the cli will check for major version updates of direct dependencies and
64/// print them to the console if the major versions are not upgradeable due to constraints.
65#[derive(Debug, Clone, Copy)]
66pub enum CheckMajorVersions {
67 Yes,
68 No,
69}
70
71pub fn list(paths: &ProjectPaths) -> Result<()> {
72 let (_, manifest) = get_manifest_details(paths)?;
73 list_manifest_packages(std::io::stdout(), manifest)
74}
75
76pub fn tree(paths: &ProjectPaths, options: TreeOptions) -> Result<()> {
77 let (config, manifest) = get_manifest_details(paths)?;
78
79 // Initialize the root package since it is not part of the manifest
80 let root_package = ManifestPackage {
81 build_tools: vec![],
82 name: config.name.clone(),
83 requirements: config.all_direct_dependencies()?.keys().cloned().collect(),
84 version: config.version.clone(),
85 source: ManifestPackageSource::Local {
86 path: paths.root().to_path_buf(),
87 },
88 otp_app: None,
89 };
90
91 // Get the manifest packages and add the root package to the vec
92 let mut packages = manifest.packages.iter().cloned().collect_vec();
93 packages.push(root_package);
94
95 list_package_and_dependencies_tree(std::io::stdout(), options, packages.clone(), config.name)
96}
97
98fn get_manifest_details(paths: &ProjectPaths) -> Result<(PackageConfig, Manifest)> {
99 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
100 let config = crate::config::root_config(paths)?;
101 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
102 let dependency_manager = DependencyManagerConfig {
103 use_manifest: UseManifest::Yes,
104 check_major_versions: CheckMajorVersions::No,
105 }
106 .into_dependency_manager(
107 runtime.handle().clone(),
108 package_fetcher,
109 cli::Reporter::new(),
110 Mode::Dev,
111 );
112 let manifest = dependency_manager
113 .resolve_versions(paths, &config, Vec::new())?
114 .manifest;
115 Ok((config, manifest))
116}
117
118fn list_manifest_packages<W: std::io::Write>(mut buffer: W, manifest: Manifest) -> Result<()> {
119 let packages = manifest
120 .packages
121 .into_iter()
122 .map(|package| vec![package.name.to_string(), package.version.to_string()])
123 .collect_vec();
124 let out = space_table(&["Package", "Version"], packages);
125
126 write!(buffer, "{out}").map_err(|e| Error::StandardIo {
127 action: StandardIoAction::Write,
128 err: Some(e.kind()),
129 })
130}
131
132fn list_package_and_dependencies_tree<W: std::io::Write>(
133 mut buffer: W,
134 options: TreeOptions,
135 packages: Vec<ManifestPackage>,
136 root_package_name: EcoString,
137) -> Result<()> {
138 let mut invert = false;
139
140 let package: Option<&ManifestPackage> = if let Some(input_package_name) = options.package {
141 packages.iter().find(|p| p.name == input_package_name)
142 } else if let Some(input_package_name) = options.invert {
143 invert = true;
144 packages.iter().find(|p| p.name == input_package_name)
145 } else {
146 packages.iter().find(|p| p.name == root_package_name)
147 };
148
149 if let Some(package) = package {
150 let tree = Vec::from([eco_format!("{0} v{1}", package.name, package.version)]);
151 let tree =
152 list_dependencies_tree(tree, package.clone(), packages, EcoString::new(), invert);
153
154 tree.iter()
155 .try_for_each(|line| writeln!(buffer, "{line}"))
156 .map_err(|e| Error::StandardIo {
157 action: StandardIoAction::Write,
158 err: Some(e.kind()),
159 })
160 } else {
161 writeln!(buffer, "Package not found. Please check the package name.").map_err(|e| {
162 Error::StandardIo {
163 action: StandardIoAction::Write,
164 err: Some(e.kind()),
165 }
166 })
167 }
168}
169
170fn list_dependencies_tree(
171 mut tree: Vec<EcoString>,
172 package: ManifestPackage,
173 packages: Vec<ManifestPackage>,
174 accum: EcoString,
175 invert: bool,
176) -> Vec<EcoString> {
177 let dependencies = packages
178 .iter()
179 .filter(|p| {
180 (invert && p.requirements.contains(&package.name))
181 || (!invert && package.requirements.contains(&p.name))
182 })
183 .cloned()
184 .collect_vec();
185
186 let dependencies = dependencies.iter().sorted().enumerate();
187
188 let deps_length = dependencies.len();
189 for (index, dependency) in dependencies {
190 let is_last = index == deps_length - 1;
191 let prefix = if is_last {
192 UTF8_SYMBOLS.ell
193 } else {
194 UTF8_SYMBOLS.tee
195 };
196
197 tree.push(eco_format!(
198 "{0}{1}{2}{2} {3} v{4}",
199 accum.clone(),
200 prefix,
201 UTF8_SYMBOLS.right,
202 dependency.name,
203 dependency.version
204 ));
205
206 let accum = accum.clone() + (if !is_last { UTF8_SYMBOLS.down } else { " " }) + " ";
207
208 tree = list_dependencies_tree(
209 tree.clone(),
210 dependency.clone(),
211 packages.clone(),
212 accum.clone(),
213 invert,
214 );
215 }
216
217 tree
218}
219
220pub fn outdated(paths: &ProjectPaths) -> Result<()> {
221 let (_, manifest) = get_manifest_details(paths)?;
222
223 let total_packages = manifest
224 .packages
225 .iter()
226 .filter(|package| matches!(package.source, ManifestPackageSource::Hex { .. }))
227 .count();
228
229 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
230 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
231
232 let version_updates = dependency::check_for_version_updates(&manifest, &package_fetcher);
233
234 print!(
235 "{}",
236 pretty_print_outdated_versions(total_packages, version_updates)
237 );
238
239 Ok(())
240}
241
242#[derive(Debug, Clone, Copy)]
243pub enum UseManifest {
244 Yes,
245 No,
246}
247
248pub fn update(paths: &ProjectPaths, packages: Vec<String>) -> Result<()> {
249 let use_manifest = if packages.is_empty() {
250 UseManifest::No
251 } else {
252 UseManifest::Yes
253 };
254
255 // Update specific packages
256 _ = resolve_and_download(
257 paths,
258 cli::Reporter::new(),
259 None,
260 packages.into_iter().map(EcoString::from).collect(),
261 DependencyManagerConfig {
262 use_manifest,
263 check_major_versions: CheckMajorVersions::Yes,
264 },
265 )?;
266
267 Ok(())
268}
269
270/// Edit the manifest.toml file in this proejct, removing all extra requirements and packages
271/// that are no longer present in the gleam.toml config.
272pub fn cleanup<Telem: Telemetry>(paths: &ProjectPaths, telemetry: Telem) -> Result<Manifest> {
273 let span = tracing::info_span!("remove_deps");
274 let _enter = span.enter();
275
276 // We do this before acquiring the build lock so that we don't create the
277 // build directory if there is no gleam.toml
278 crate::config::ensure_config_exists(paths)?;
279
280 let lock = BuildLock::new_packages(paths)?;
281 let _guard: Guard = lock.lock(&telemetry)?;
282
283 // Read the project config
284 let config = crate::root_config(paths)?;
285 let old_manifest = read_manifest_from_disc(paths)?;
286 let mut manifest = old_manifest.clone();
287
288 remove_extra_requirements(&config, &mut manifest)?;
289
290 // Remove any packages that are no longer required due to manifest changes
291 let local = LocalPackages::read_from_disc(paths)?;
292 remove_extra_packages(paths, &local, &manifest, &telemetry)?;
293
294 // Record new state of the packages directory
295 tracing::debug!("writing_manifest_toml");
296 write_manifest_to_disc(paths, &manifest)?;
297 LocalPackages::from_manifest(&manifest).write_to_disc(paths)?;
298
299 let changes = PackageChanges::between_manifests(&old_manifest, &manifest);
300 telemetry.resolved_package_versions(&changes);
301
302 // Cleanup build cache of the root package if there are some changes.
303 // Without this, if a removed dependency is still used, teh build will
304 // succeed, resulting in runtime crash due to missing files.
305 if changes.any_changes() {
306 tracing::debug!("cleaning_root_package_build_cache");
307 for mode in Mode::iter() {
308 for target in Target::iter() {
309 let lock = BuildLock::new_target(paths, mode, target)?;
310 let _guard = lock.lock(&telemetry)?;
311
312 let build_directory_path =
313 paths.build_directory_for_package(mode, target, &config.name);
314 if build_directory_path.exists() {
315 fs::delete_directory(build_directory_path.as_ref())?;
316 }
317 }
318 }
319 }
320
321 Ok(manifest)
322}
323
324/// Remove requirements and unneeded packages from manifest that are no longer present in config.
325fn remove_extra_requirements(config: &PackageConfig, manifest: &mut Manifest) -> Result<()> {
326 // "extra requirements" are all packages that are requirements in the manifest, but no longer
327 // part of the gleam.toml config.
328 let is_extra_requirement = |name: &EcoString| {
329 !config.dev_dependencies.contains_key(name) && !config.dependencies.contains_key(name)
330 };
331
332 // If a requirement is also used as a dependency, we do not want to force-unlock it.
333 // If the dependents get deleted as well, this transitive dependency will be dropped.
334 let is_unlockable_requirement = |name: &EcoString| {
335 manifest
336 .packages
337 .iter()
338 .all(|p| !p.requirements.contains(name))
339 };
340
341 let extra_requirements = manifest
342 .requirements
343 .keys()
344 .filter(|&name| is_extra_requirement(name) && is_unlockable_requirement(name))
345 .cloned()
346 .collect::<Vec<_>>();
347
348 manifest
349 .requirements
350 .retain(|name, _| !is_extra_requirement(name));
351
352 // Unlock all packages that we we want to remove - this removes them and all unneeded
353 // dependencies from `locked`.
354 let mut locked = config.locked(Some(manifest))?;
355 unlock_packages(&mut locked, extra_requirements.as_slice(), Some(manifest))?;
356 // Remove all unlocked packages from the manifest - these are truly no longer needed.
357 manifest
358 .packages
359 .retain(|package| locked.contains_key(&package.name));
360
361 Ok(())
362}
363
364pub fn parse_gleam_add_specifier(package: &str) -> Result<(EcoString, Requirement)> {
365 let Some((package, version)) = package.split_once('@') else {
366 // Default to the latest version available.
367 return Ok((
368 package.into(),
369 Requirement::hex(">= 0.0.0").expect("'>= 0.0.0' should be a valid pubgrub range"),
370 ));
371 };
372
373 // Parse the major and minor from the provided semantic version.
374 let parts = version.split('.').collect::<Vec<_>>();
375 let major = match parts.first() {
376 Some(major) => Ok(major),
377 None => Err(Error::InvalidVersionFormat {
378 input: package.to_string(),
379 error: "Failed to parse semantic major version".to_string(),
380 }),
381 }?;
382 let minor = match parts.get(1) {
383 Some(minor) => minor,
384 None => "0",
385 };
386
387 // Using the major version specifier, calculate the maximum
388 // allowable version (i.e., the next major version).
389 let Ok(num) = major.parse::<usize>() else {
390 return Err(Error::InvalidVersionFormat {
391 input: version.to_string(),
392 error: "Failed to parse semantic major version as integer".to_string(),
393 });
394 };
395
396 let max_ver = [&(num + 1).to_string(), "0", "0"].join(".");
397
398 // Pad the provided version specifier with zeros map to a Hex version.
399 let requirement = match parts.len() {
400 1 | 2 => {
401 let min_ver = [major, minor, "0"].join(".");
402 Requirement::hex(&[">=", &min_ver, "and", "<", &max_ver].join(" "))
403 }
404 3 => Requirement::hex(version),
405 n => {
406 return Err(Error::InvalidVersionFormat {
407 input: version.to_string(),
408 error: format!(
409 "Expected up to 3 numbers in version specifier (MAJOR.MINOR.PATCH), found {n}"
410 ),
411 });
412 }
413 }?;
414
415 Ok((package.into(), requirement))
416}
417
418pub fn resolve_and_download<Telem: Telemetry>(
419 paths: &ProjectPaths,
420 telemetry: Telem,
421 new_package: Option<(Vec<(EcoString, Requirement)>, bool)>,
422 packages_to_update: Vec<EcoString>,
423 config: DependencyManagerConfig,
424) -> Result<Manifest> {
425 // Start event loop so we can run async functions to call the Hex API
426 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
427 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
428
429 let dependency_manager = config.into_dependency_manager(
430 runtime.handle().clone(),
431 package_fetcher,
432 telemetry,
433 Mode::Dev,
434 );
435
436 dependency_manager.resolve_and_download_versions(paths, new_package, packages_to_update)
437}
438
439fn format_versions_and_extract_longest_parts(
440 versions: dependency::PackageVersionDiffs,
441) -> Vec<Vec<String>> {
442 versions
443 .iter()
444 .map(|(name, (v1, v2))| vec![name.to_string(), v1.to_string(), v2.to_string()])
445 .sorted()
446 .collect_vec()
447}
448
449fn pretty_print_major_versions_available(versions: dependency::PackageVersionDiffs) -> String {
450 let versions = format_versions_and_extract_longest_parts(versions);
451
452 format!(
453 "\nThe following dependencies have new major versions available:\n\n{}",
454 space_table(&["Package", "Current", "Latest"], &versions)
455 )
456}
457
458fn pretty_print_version_updates(versions: dependency::PackageVersionDiffs) -> EcoString {
459 let versions = format_versions_and_extract_longest_parts(versions);
460 space_table(&["Package", "Current", "Latest"], &versions)
461}
462
463fn pretty_print_outdated_versions(
464 total_packages: usize,
465 versions: dependency::PackageVersionDiffs,
466) -> EcoString {
467 let summary = eco_format!(
468 "{} of {} packages have newer versions available.",
469 versions.len(),
470 total_packages
471 );
472
473 if versions.is_empty() {
474 eco_format!("{}\n", summary)
475 } else {
476 eco_format!("{}\n\n{}", summary, pretty_print_version_updates(versions))
477 }
478}
479
480async fn add_missing_packages<Telem: Telemetry>(
481 paths: &ProjectPaths,
482 fs: Box<ProjectIO>,
483 manifest: &Manifest,
484 local: &LocalPackages,
485 project_name: EcoString,
486 telemetry: &Telem,
487) -> Result<(), Error> {
488 let missing_packages = local.missing_local_packages(manifest, &project_name);
489
490 let mut num_to_download = 0;
491
492 let missing_git_packages = missing_packages
493 .iter()
494 .copied()
495 .filter(|package| package.is_git())
496 .inspect(|_| {
497 num_to_download += 1;
498 })
499 .collect_vec();
500
501 let mut missing_hex_packages = missing_packages
502 .iter()
503 .copied()
504 .filter(|package| package.is_hex())
505 .inspect(|_| {
506 num_to_download += 1;
507 })
508 .peekable();
509
510 // If we need to download at-least one package
511 if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() {
512 let http = HttpClient::boxed();
513 let downloader = hex::Downloader::new(
514 fs.clone(),
515 fs,
516 http,
517 Untar::boxed(),
518 crate::hex::read_env_readonly_api_key(),
519 paths.clone(),
520 );
521 let start = Instant::now();
522 telemetry.downloading_package("packages");
523 downloader
524 .download_hex_packages(missing_hex_packages, &project_name)
525 .await?;
526 for package in missing_git_packages {
527 let ManifestPackageSource::Git { repo, commit, path } = &package.source else {
528 continue;
529 };
530
531 let checkout =
532 download_git_package(&package.name, repo, commit, path.as_deref(), paths)?;
533 checkout.cleanup()?;
534 }
535 telemetry.packages_downloaded(start, num_to_download);
536 }
537
538 Ok(())
539}
540
541fn remove_extra_packages<Telem: Telemetry>(
542 paths: &ProjectPaths,
543 local: &LocalPackages,
544 manifest: &Manifest,
545 telemetry: &Telem,
546) -> Result<()> {
547 let _guard = BuildLock::lock_all_build(paths, telemetry)?;
548
549 for (package_name, version) in local.extra_local_packages(manifest) {
550 // TODO: test
551 // Delete the package source
552 let path = paths.build_packages_package(&package_name);
553 if path.exists() {
554 tracing::debug!(package=%package_name, version=%version, "removing_unneeded_package");
555 fs::delete_directory(&path)?;
556 }
557
558 // TODO: test
559 // Delete any build artefacts for the package
560 for mode in Mode::iter() {
561 for target in Target::iter() {
562 let name = manifest
563 .packages
564 .iter()
565 .find(|p| p.name == package_name)
566 .map(|p| p.application_name().as_str())
567 .unwrap_or(package_name.as_str());
568 let path = paths.build_directory_for_package(mode, target, name);
569 if path.exists() {
570 tracing::debug!(package=%package_name, version=%version, "deleting_build_cache");
571 fs::delete_directory(&path)?;
572 }
573 }
574 }
575 }
576
577 remove_unused_git_clones(paths, manifest)?;
578 Ok(())
579}
580
581fn remove_unused_git_clones(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
582 let git_directory = paths.build_git_directory();
583 if !git_directory.is_dir() {
584 return Ok(());
585 }
586
587 let expected: HashSet<String> = manifest
588 .packages
589 .iter()
590 .filter_map(|package| match &package.source {
591 ManifestPackageSource::Git {
592 repo,
593 path: Some(_),
594 ..
595 } => Some(git_repo_dir_name(repo)),
596 _ => None,
597 })
598 .collect();
599
600 for entry in fs::read_dir(&git_directory)?.filter_map(Result::ok) {
601 if !expected.contains(entry.file_name()) {
602 tracing::debug!(path=%entry.path(), "removing_unused_git_clone");
603 fs::delete_directory(entry.path())?;
604 }
605 }
606 Ok(())
607}
608
609fn read_manifest_from_disc(paths: &ProjectPaths) -> Result<Manifest> {
610 tracing::debug!("reading_manifest_toml");
611 let manifest_path = paths.manifest();
612 let toml = fs::read(&manifest_path)?;
613 let manifest = toml::from_str(&toml).map_err(|e| Error::FileIo {
614 action: FileIoAction::Parse,
615 kind: FileKind::File,
616 path: manifest_path.clone(),
617 err: Some(e.to_string()),
618 })?;
619 Ok(manifest)
620}
621
622fn write_manifest_to_disc(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
623 let path = paths.manifest();
624 fs::write(&path, &manifest.to_toml(paths.root()))
625}
626
627// This is the container for locally pinned packages, representing the current contents of
628// the `project/build/packages` directory.
629// For descriptions of packages provided by paths and git deps, see the ProvidedPackage struct.
630// The same package may appear in both at different times.
631#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
632struct LocalPackages {
633 #[serde(deserialize_with = "gleam_core::config::map_with_package_name_keys::deserialize")]
634 packages: HashMap<EcoString, Version>,
635 // Git packages can resolve to a new commit (or a new sub-path) without
636 // their version changing, so their on-disc freshness is keyed by commit
637 // and path rather than version alone. Absent for hex and local packages.
638 #[serde(default)]
639 git: HashMap<EcoString, GitState>,
640}
641
642#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
643struct GitState {
644 commit: EcoString,
645 #[serde(default)]
646 path: Option<Utf8PathBuf>,
647}
648
649impl LocalPackages {
650 pub fn extra_local_packages(&self, manifest: &Manifest) -> Vec<(EcoString, Version)> {
651 let manifest_packages: HashSet<_> = manifest
652 .packages
653 .iter()
654 .map(|p| (&p.name, &p.version))
655 .collect();
656 self.packages
657 .iter()
658 .filter(|(n, v)| !manifest_packages.contains(&(&EcoString::from(*n), v)))
659 .map(|(n, v)| (n.clone(), v.clone()))
660 .collect()
661 }
662
663 pub fn missing_local_packages<'a>(
664 &self,
665 manifest: &'a Manifest,
666 root: &str,
667 ) -> Vec<&'a ManifestPackage> {
668 manifest
669 .packages
670 .iter()
671 // We don't need to download the root package
672 .filter(|p| p.name != root)
673 // We don't need to download local packages because we use the linked source directly
674 .filter(|p| !p.is_local())
675 // We don't need to download packages which we already have on disc. For
676 // git packages this means the same commit and path.
677 .filter(|p| !self.has_fresh(p))
678 .collect()
679 }
680
681 fn has_fresh(&self, package: &ManifestPackage) -> bool {
682 match &package.source {
683 ManifestPackageSource::Git { commit, path, .. } => match self.git.get(&package.name) {
684 Some(state) => &state.commit == commit && &state.path == path,
685 None => false,
686 },
687 ManifestPackageSource::Hex { .. } | ManifestPackageSource::Local { .. } => {
688 self.packages.get(package.name.as_str()) == Some(&package.version)
689 }
690 }
691 }
692
693 pub fn read_from_disc(paths: &ProjectPaths) -> Result<Self> {
694 let path = paths.build_packages_toml();
695 if !path.exists() {
696 return Ok(Self {
697 packages: HashMap::new(),
698 git: HashMap::new(),
699 });
700 }
701 let toml = fs::read(&path)?;
702 toml::from_str(&toml).map_err(|e| Error::FileIo {
703 action: FileIoAction::Parse,
704 kind: FileKind::File,
705 path: path.clone(),
706 err: Some(e.to_string()),
707 })
708 }
709
710 pub fn write_to_disc(&self, paths: &ProjectPaths) -> Result<()> {
711 let path = paths.build_packages_toml();
712 let toml = toml::to_string(&self).expect("packages.toml serialization");
713 fs::write(&path, &toml)
714 }
715
716 pub fn from_manifest(manifest: &Manifest) -> Self {
717 let packages = manifest
718 .packages
719 .iter()
720 .map(|p| (p.name.clone(), p.version.clone()))
721 .collect();
722 let git = manifest
723 .packages
724 .iter()
725 .filter_map(|p| match &p.source {
726 ManifestPackageSource::Git { commit, path, .. } => Some((
727 p.name.clone(),
728 GitState {
729 commit: commit.clone(),
730 path: path.clone(),
731 },
732 )),
733 ManifestPackageSource::Hex { .. } | ManifestPackageSource::Local { .. } => None,
734 })
735 .collect();
736
737 Self { packages, git }
738 }
739}
740
741#[test]
742fn local_packages_deserialise_ok() {
743 let toml = r#"
744[packages]
745gleam_stdlib = "1.0.0"
746gleam_otp = "1.1.0"
747"#;
748 let packages: LocalPackages = toml::from_str(toml).unwrap();
749 assert_eq!(
750 packages,
751 LocalPackages {
752 packages: HashMap::from_iter([
753 ("gleam_stdlib".into(), Version::new(1, 0, 0)),
754 ("gleam_otp".into(), Version::new(1, 1, 0)),
755 ]),
756 git: HashMap::new(),
757 }
758 )
759}
760
761#[test]
762fn local_packages_deserialise_invalid_name() {
763 let toml = r#"
764[packages]
765gleam_stdlib = "1.0.0"
766"../../stuff" = "1.1.0"
767"#;
768 let error = toml::from_str::<LocalPackages>(toml)
769 .expect_err("should fail to deserialise because of invalid name");
770 insta::assert_snapshot!(insta::internals::AutoName, error.to_string());
771}
772
773fn is_same_requirements(
774 requirements1: &HashMap<EcoString, Requirement>,
775 requirements2: &HashMap<EcoString, Requirement>,
776 root_path: &Utf8Path,
777) -> Result<bool> {
778 if requirements1.len() != requirements2.len() {
779 return Ok(false);
780 }
781
782 for (key, requirement1) in requirements1 {
783 if !same_requirements(requirement1, requirements2.get(key), root_path)? {
784 return Ok(false);
785 }
786 }
787
788 Ok(true)
789}
790
791/// Returns true if all path dependency configs are unchanged since last build.
792///
793/// If any of the path dependency configs have changed that means that we need to
794/// re-perform dependency resolution, as their dependencies could have changed
795/// themselves.
796///
797/// We use gleam.toml rather than manifest.toml as:
798///
799/// 1. The dependency requirements could have changed but resolution not have been
800/// run in that package yet, so the manifest would be the same, resulting in us
801/// failing to detect that resolution is required.
802///
803/// 2. Dependency manifests are not used in any way, so a change in the manifest
804/// may not have any impact on this package.
805///
806/// This does mean that changes unrelated to the path dependency's dependencies
807/// will trigger resolution, but gleam.toml is edited rarely, and no-change
808/// resolution is fast enough, so that's OK.
809///
810/// Note: This does not check path dependencies of path dependencies! Changes to
811/// their configs will fail to be picked up. To resolve this we would need to keep
812/// a list of all the path dependencies in the project, instead of only the direct
813/// path dependencies.
814///
815fn path_dependency_configs_unchanged(
816 requirements: &HashMap<EcoString, Requirement>,
817 paths: &ProjectPaths,
818) -> Result<bool> {
819 for (name, requirement) in requirements {
820 let Requirement::Path { path } = requirement else {
821 continue;
822 };
823
824 let config_path = paths.path_dependency_gleam_toml_path(path);
825 let fingerprint_path = paths.dependency_gleam_toml_fingerprint_path(name.as_str());
826
827 // Check mtimes before hashing, to avoid extra work
828 if fingerprint_path.exists() {
829 let config_time = fs::modification_time(&config_path)?;
830 let fingerprint_time = fs::modification_time(&fingerprint_path)?;
831 if config_time <= fingerprint_time {
832 continue;
833 }
834 };
835
836 let config_text = fs::read(&config_path)?;
837 let current_fingerprint = SourceFingerprint::new(&config_text).to_numerical_string();
838
839 // If cached hash file doesn't exist, this is the first time we're checking this dependency
840 if !fingerprint_path.exists() {
841 // Save the current hash for future comparisons
842 fs::write(&fingerprint_path, ¤t_fingerprint)?;
843 return Ok(false);
844 }
845
846 let previous_fingerprint = fs::read(&fingerprint_path)?;
847
848 if previous_fingerprint != current_fingerprint {
849 tracing::debug!("path_dependency_config_changed_forcing_rebuild");
850 fs::write(&fingerprint_path, ¤t_fingerprint)?;
851 return Ok(false);
852 }
853 }
854
855 Ok(true)
856}
857
858fn same_requirements(
859 requirement1: &Requirement,
860 requirement2: Option<&Requirement>,
861 root_path: &Utf8Path,
862) -> Result<bool> {
863 let (left, right) = match (requirement1, requirement2) {
864 (Requirement::Path { path: path1 }, Some(Requirement::Path { path: path2 })) => {
865 let left = fs::canonicalise(&root_path.join(path1))?;
866 let right = fs::canonicalise(&root_path.join(path2))?;
867 (left, right)
868 }
869 (_, Some(requirement2)) => return Ok(requirement1 == requirement2),
870 (_, None) => return Ok(false),
871 };
872
873 Ok(left == right)
874}
875
876#[derive(Clone, Eq, PartialEq, Debug)]
877struct ProvidedPackage {
878 version: Version,
879 source: ProvidedPackageSource,
880 requirements: HashMap<EcoString, hexpm::version::Range>,
881}
882
883#[derive(Clone, Eq, Debug)]
884enum ProvidedPackageSource {
885 Git {
886 repo: EcoString,
887 commit: EcoString,
888 path: Option<Utf8PathBuf>,
889 },
890 Local {
891 path: Utf8PathBuf,
892 },
893}
894
895impl ProvidedPackage {
896 fn to_hex_package(&self, name: &EcoString) -> hexpm::Package {
897 let requirements = self
898 .requirements
899 .iter()
900 .map(|(name, version)| {
901 (
902 name.as_str().into(),
903 hexpm::Dependency {
904 requirement: version.clone(),
905 optional: false,
906 app: None,
907 repository: None,
908 },
909 )
910 })
911 .collect();
912 let release = hexpm::Release {
913 version: self.version.clone(),
914 requirements,
915 retirement_status: None,
916 outer_checksum: vec![],
917 meta: (),
918 };
919 hexpm::Package {
920 name: name.as_str().into(),
921 repository: "local".into(),
922 releases: vec![release],
923 }
924 }
925
926 fn to_manifest_package(&self, name: &str) -> ManifestPackage {
927 let mut package = ManifestPackage {
928 name: name.into(),
929 version: self.version.clone(),
930 otp_app: None, // Note, this will probably need to be set to something eventually
931 build_tools: vec!["gleam".into()],
932 requirements: self.requirements.keys().cloned().collect(),
933 source: self.source.to_manifest_package_source(),
934 };
935 package.requirements.sort();
936 package
937 }
938}
939
940impl ProvidedPackageSource {
941 fn to_manifest_package_source(&self) -> ManifestPackageSource {
942 match self {
943 Self::Git { repo, commit, path } => ManifestPackageSource::Git {
944 repo: repo.clone(),
945 commit: commit.clone(),
946 path: path.clone(),
947 },
948 Self::Local { path } => ManifestPackageSource::Local { path: path.clone() },
949 }
950 }
951
952 fn to_toml(&self) -> String {
953 match self {
954 Self::Git { repo, commit, path } => match path {
955 Some(path) => {
956 format!(r#"{{ repo: "{repo}", commit: "{commit}", path: "{path}" }}"#)
957 }
958 None => format!(r#"{{ repo: "{repo}", commit: "{commit}" }}"#),
959 },
960 Self::Local { path } => {
961 format!(r#"{{ path: "{path}" }}"#)
962 }
963 }
964 }
965}
966
967impl PartialEq for ProvidedPackageSource {
968 fn eq(&self, other: &Self) -> bool {
969 match (self, other) {
970 (Self::Local { path: own_path }, Self::Local { path: other_path }) => {
971 is_same_file(own_path, other_path).unwrap_or(false)
972 }
973
974 (
975 Self::Git {
976 repo: own_repo,
977 commit: own_commit,
978 path: own_path,
979 },
980 Self::Git {
981 repo: other_repo,
982 commit: other_commit,
983 path: other_path,
984 },
985 ) => own_repo == other_repo && own_commit == other_commit && own_path == other_path,
986
987 (Self::Git { .. }, Self::Local { .. }) | (Self::Local { .. }, Self::Git { .. }) => {
988 false
989 }
990 }
991 }
992}
993
994/// Where a provided package came from. Git sources carry the on-disc repository
995/// root so path dependencies can be resolved relative to it.
996enum SourceContext<'a> {
997 Local {
998 path: Utf8PathBuf,
999 },
1000 Git {
1001 repo: EcoString,
1002 commit: EcoString,
1003 path: Option<Utf8PathBuf>,
1004 repo_root: &'a Utf8Path,
1005 },
1006}
1007
1008impl SourceContext<'_> {
1009 fn to_provided_source(&self) -> ProvidedPackageSource {
1010 match self {
1011 Self::Local { path } => ProvidedPackageSource::Local { path: path.clone() },
1012 Self::Git {
1013 repo, commit, path, ..
1014 } => ProvidedPackageSource::Git {
1015 repo: repo.clone(),
1016 commit: commit.clone(),
1017 path: path.clone(),
1018 },
1019 }
1020 }
1021}
1022
1023/// Provide a package from a local project
1024fn provide_local_package(
1025 package_name: EcoString,
1026 package_path: &Utf8Path,
1027 parent_path: &Utf8Path,
1028 project_paths: &ProjectPaths,
1029 provided: &mut HashMap<EcoString, ProvidedPackage>,
1030 parents: &mut Vec<EcoString>,
1031) -> Result<hexpm::version::Range> {
1032 let package_path = if package_path.is_absolute() {
1033 package_path.to_path_buf()
1034 } else {
1035 fs::canonicalise(&parent_path.join(package_path))?
1036 };
1037
1038 provide_package(
1039 package_name,
1040 package_path.clone(),
1041 SourceContext::Local { path: package_path },
1042 project_paths,
1043 provided,
1044 parents,
1045 )
1046}
1047
1048/// Resolve a path dependency of a git package to its canonical filesystem
1049/// location and its repository-relative path.
1050fn resolve_git_path_package(
1051 package_name: &EcoString,
1052 path: &Utf8Path,
1053 repo: &EcoString,
1054 parent_path: &Utf8Path,
1055 repo_root: &Utf8Path,
1056) -> Result<(Utf8PathBuf, Utf8PathBuf)> {
1057 let location = parent_path.join(path);
1058 if !location.is_dir() {
1059 return Err(Error::GitDependencyPathNotFound {
1060 package: package_name.to_string(),
1061 path: path.to_string(),
1062 repo: repo.to_string(),
1063 });
1064 }
1065
1066 // The path may name a symlink that points outside the repository
1067 // checkout, so resolve it and take the repository-relative path from the
1068 // canonical location.
1069 let package_path = fs::canonicalise(&location)?;
1070 let repo_path = package_path
1071 .strip_prefix(repo_root)
1072 .map_err(|_| Error::GitDependencyPathNotFound {
1073 package: package_name.to_string(),
1074 path: path.to_string(),
1075 repo: repo.to_string(),
1076 })?
1077 .to_path_buf();
1078
1079 Ok((package_path, repo_path))
1080}
1081
1082fn execute_command(command: &mut Command) -> Result<std::process::Output> {
1083 let result = command.output();
1084 match result {
1085 Ok(output) if output.status.success() => Ok(output),
1086 Ok(output) => {
1087 let reason = match String::from_utf8(output.stderr) {
1088 Ok(stderr) => ShellCommandFailureReason::ShellCommandError(stderr),
1089 Err(_) => ShellCommandFailureReason::Unknown,
1090 };
1091 Err(Error::ShellCommand {
1092 program: "git".into(),
1093 reason,
1094 })
1095 }
1096 Err(error) => Err(git_io_error(error)),
1097 }
1098}
1099
1100/// A `git` command with its working directory set to `dir`.
1101fn git_command(dir: &Utf8Path) -> Command {
1102 let mut command = Command::new("git");
1103 let _ = command.current_dir(dir);
1104 command
1105}
1106
1107/// Map an IO error from spawning `git` onto the appropriate `Error`.
1108fn git_io_error(error: std::io::Error) -> Error {
1109 match error.kind() {
1110 ErrorKind::NotFound => Error::ShellProgramNotFound {
1111 program: "git".into(),
1112 os: fs::get_os(),
1113 },
1114 other => Error::ShellCommand {
1115 program: "git".into(),
1116 reason: ShellCommandFailureReason::IoError(other),
1117 },
1118 }
1119}
1120
1121/// Parse the trimmed UTF-8 stdout of a `git` command.
1122fn git_stdout(output: std::process::Output) -> EcoString {
1123 String::from_utf8(output.stdout)
1124 .expect("Output should be UTF-8")
1125 .trim()
1126 .into()
1127}
1128
1129fn git_repo_dir_name(repo: &str) -> String {
1130 let name = repo
1131 .trim_end_matches('/')
1132 .trim_end_matches(".git")
1133 .rsplit(['/', ':', '\\'])
1134 .next()
1135 .filter(|name| !name.is_empty())
1136 .unwrap_or("repo");
1137 let hash = xxhash_rust::xxh3::xxh3_64(repo.as_bytes());
1138 format!("{name}-{hash:016x}")
1139}
1140
1141fn git_staging_path(project_paths: &ProjectPaths, repo: &str, package_name: &str) -> Utf8PathBuf {
1142 project_paths.build_git_repo(&format!(
1143 "{}-{package_name}-staging",
1144 git_repo_dir_name(repo)
1145 ))
1146}
1147
1148enum GitCheckout {
1149 InPlace {
1150 commit: EcoString,
1151 },
1152 Staged {
1153 commit: EcoString,
1154 staging_path: Utf8PathBuf,
1155 },
1156}
1157
1158impl GitCheckout {
1159 fn cleanup(self) -> Result<()> {
1160 // The clones dangling worktree registration is left for the next
1161 // download to prune before it re-adds the staging worktree.
1162 if let Self::Staged { staging_path, .. } = self {
1163 fs::delete_directory(&staging_path)?;
1164 }
1165 Ok(())
1166 }
1167}
1168
1169/// Downloads a git package from a remote repository.
1170///
1171/// For a package at the root of its repository the commands that are run look
1172/// like this, cloning directly into `build/packages/<name>`:
1173///
1174/// ```sh
1175/// git init
1176/// git remote remove origin
1177/// git remote add origin <repo>
1178/// git fetch origin
1179/// git checkout <ref>
1180/// git rev-parse HEAD
1181/// ```
1182///
1183/// For a package in a subdirectory of its repository the repository is
1184/// instead cloned into `build/git/<repo-name>-<hash>` as a bare repository
1185/// with no work tree, and the package is checked out into a transient staging
1186/// worktree which the subdirectory is hard-linked out of:
1187///
1188/// ```sh
1189/// git init --bare
1190/// git remote remove origin
1191/// git remote add origin <repo>
1192/// git fetch origin
1193/// git rev-parse --verify --quiet <ref>^{commit}
1194/// git worktree prune
1195/// git worktree add --force --detach <clone>-<pkg>-staging <commit>
1196/// ```
1197///
1198/// The staging worktree exists only for the duration of one package download.
1199/// It is deleted (and the clone's worktree registration pruned) by
1200/// `GitCheckout::cleanup` once the package has been hard-linked and read.
1201///
1202/// This is somewhat inefficient as we have to fetch the entire git history before
1203/// switching to the exact commit we want. There a few alternatives to this:
1204///
1205/// - `git clone --depth 1 --branch="<ref>"` This works, but only allows us to use
1206/// branch names as refs, however we want to allow commit hashes as well.
1207/// - `git fetch --depth 1 origin <ref>` Similarly, this imposes an unwanted
1208/// restriction. `git fetch` only allows branch names or full commit hashes,
1209/// but we want to allow partial hashes as well.
1210///
1211/// Since Git dependencies will be used quite rarely, this option was settled upon
1212/// because it allows branch names, full and partial commit hashes as refs.
1213///
1214/// In the future we can optimise this more, for example first checking if we
1215/// are already checked out to the commit stored in the manifest, or by only
1216/// fetching the history without the objects to resolve partial commit hashes.
1217/// For now though this is good enough until it become an actual performance
1218/// problem.
1219///
1220fn download_git_package(
1221 package_name: &str,
1222 repo: &str,
1223 ref_: &str,
1224 path: Option<&Utf8Path>,
1225 project_paths: &ProjectPaths,
1226) -> Result<GitCheckout> {
1227 match path {
1228 None => download_git_package_in_place(package_name, repo, ref_, project_paths),
1229 Some(subdir) => {
1230 download_git_package_to_staged_path(package_name, repo, ref_, project_paths, subdir)
1231 }
1232 }
1233}
1234
1235fn download_git_package_in_place(
1236 package_name: &str,
1237 repo: &str,
1238 ref_: &str,
1239 project_paths: &ProjectPaths,
1240) -> Result<GitCheckout, Error> {
1241 let clone_path = project_paths.build_packages_package(package_name);
1242 prepare_git_clone(&clone_path, repo, CloneFormat::WorkTree)?;
1243 let _ = execute_command(git_command(&clone_path).arg("checkout").arg(ref_))?;
1244 let output = execute_command(git_command(&clone_path).arg("rev-parse").arg("HEAD"))?;
1245 let commit = git_stdout(output);
1246
1247 Ok(GitCheckout::InPlace { commit })
1248}
1249
1250fn download_git_package_to_staged_path(
1251 package_name: &str,
1252 repo: &str,
1253 ref_: &str,
1254 project_paths: &ProjectPaths,
1255 subdir: &Utf8Path,
1256) -> Result<GitCheckout, Error> {
1257 let clone_path = project_paths.build_git_repo(&git_repo_dir_name(repo));
1258 prepare_git_clone(&clone_path, repo, CloneFormat::Bare)?;
1259
1260 let commit = resolve_git_ref(&clone_path, repo, ref_)?;
1261
1262 // Delete any staging worktree left behind by a previous crash, and prune
1263 // its registration from the clone so `git worktree add` can reuse the
1264 // path.
1265 let staging_path = git_staging_path(project_paths, repo, package_name);
1266 fs::delete_directory(&staging_path)?;
1267 let _ = git_command(&clone_path)
1268 .arg("worktree")
1269 .arg("prune")
1270 .output();
1271
1272 let _ = execute_command(
1273 git_command(&clone_path)
1274 .arg("worktree")
1275 .arg("add")
1276 .arg("--force")
1277 .arg("--detach")
1278 .arg(&staging_path)
1279 .arg(commit.as_str()),
1280 )?;
1281
1282 let Some(subdir_source) = resolve_git_subdir(&staging_path, subdir) else {
1283 return Err(Error::GitDependencyPathNotFound {
1284 package: package_name.into(),
1285 path: subdir.to_string(),
1286 repo: repo.into(),
1287 });
1288 };
1289
1290 let package_path = project_paths.build_packages_package(package_name);
1291 fs::delete_directory(&package_path)?;
1292 fs::mkdir(&package_path)?;
1293 fs::hardlink_dir(&subdir_source, &package_path)?;
1294
1295 Ok(GitCheckout::Staged {
1296 commit,
1297 staging_path,
1298 })
1299}
1300
1301#[derive(Clone, Copy)]
1302enum CloneFormat {
1303 /// A regular clone with a checked-out work tree.
1304 WorkTree,
1305 /// A bare repository with no work tree.
1306 Bare,
1307}
1308
1309/// Initialise (or reuse) a git clone at the given path and fetch from the
1310/// remote repository.
1311fn prepare_git_clone(clone_path: &Utf8Path, repo: &str, format: CloneFormat) -> Result<()> {
1312 // If the clone path exists but is not the kind of git repo we expect, we
1313 // need to remove the directory.
1314 let reusable = match format {
1315 CloneFormat::Bare => fs::is_bare_git_repo_root(clone_path),
1316 CloneFormat::WorkTree => fs::is_git_work_tree_root(clone_path),
1317 };
1318
1319 if !reusable {
1320 fs::delete_directory(clone_path)?;
1321 }
1322
1323 fs::mkdir(clone_path)?;
1324
1325 let mut init = git_command(clone_path);
1326 let _ = init.arg("init");
1327 if matches!(format, CloneFormat::Bare) {
1328 let _ = init.arg("--bare");
1329 }
1330 let _ = execute_command(&mut init)?;
1331
1332 // If this directory already exists, but the remote URL has been edited in
1333 // `gleam.toml` without a `gleam clean`, `git remote add` will fail, causing
1334 // the remote to be stuck as the original value. Here we remove the remote
1335 // first, which ensures that `git remote add` properly add the remote each
1336 // time. If this fails, that means we haven't set the remote in the first
1337 // place, so we can safely ignore the error.
1338 let _ = git_command(clone_path)
1339 .arg("remote")
1340 .arg("remove")
1341 .arg("origin")
1342 .output();
1343
1344 let _ = execute_command(
1345 git_command(clone_path)
1346 .arg("remote")
1347 .arg("add")
1348 .arg("origin")
1349 .arg(repo),
1350 )?;
1351
1352 let _ = execute_command(git_command(clone_path).arg("fetch").arg("origin"))?;
1353
1354 Ok(())
1355}
1356
1357/// Resolve a ref (a branch name, tag, or full or partial commit hash) to a
1358/// full commit hash using the objects fetched into the clone, without
1359/// checking anything out. Branch names only exist as remote-tracking refs in
1360/// the never-checked-out clone, so when the ref does not resolve directly we
1361/// fall back to `origin/<ref>`.
1362fn resolve_git_ref(clone_path: &Utf8Path, repo: &str, ref_: &str) -> Result<EcoString> {
1363 let revisions = [
1364 format!("{ref_}^{{commit}}"),
1365 format!("origin/{ref_}^{{commit}}"),
1366 ];
1367
1368 for revision in revisions {
1369 let result = git_command(clone_path)
1370 .arg("rev-parse")
1371 .arg("--verify")
1372 .arg("--quiet")
1373 .arg(&revision)
1374 .output();
1375
1376 match result {
1377 // A failed rev-parse is expected when the ref form doesn't match
1378 // this pattern, so try the next one.
1379 Ok(output) if !output.status.success() => (),
1380 Ok(output) => return Ok(git_stdout(output)),
1381 Err(error) => return Err(git_io_error(error)),
1382 }
1383 }
1384
1385 Err(Error::ShellCommand {
1386 program: "git".into(),
1387 reason: ShellCommandFailureReason::ShellCommandError(format!(
1388 "Unable to resolve git ref `{ref_}` for repository `{repo}`\n"
1389 )),
1390 })
1391}
1392
1393/// Resolves a subdirectory within a cloned git repository, ensuring that the
1394/// resolved directory (after following any symlinks) is still inside the
1395/// repository checkout. Returns `None` if the directory does not exist or
1396/// escapes the checkout.
1397fn resolve_git_subdir(clone_path: &Utf8Path, subdir: &Utf8Path) -> Option<Utf8PathBuf> {
1398 let subdir_source = clone_path.join(subdir);
1399 if !subdir_source.is_dir() {
1400 return None;
1401 }
1402
1403 let canonical_clone = fs::canonicalise(clone_path).ok()?;
1404 let canonical_subdir = fs::canonicalise(&subdir_source).ok()?;
1405 if !canonical_subdir.starts_with(&canonical_clone) {
1406 return None;
1407 }
1408
1409 Some(canonical_subdir)
1410}
1411
1412/// Provide a package from a git repository
1413fn provide_git_package(
1414 package_name: EcoString,
1415 repo: &str,
1416 // A git ref, such as a branch name, commit hash or tag name
1417 ref_: &str,
1418 path: Option<Utf8PathBuf>,
1419 project_paths: &ProjectPaths,
1420 provided: &mut HashMap<EcoString, ProvidedPackage>,
1421 parents: &mut Vec<EcoString>,
1422) -> Result<hexpm::version::Range> {
1423 let checkout = download_git_package(&package_name, repo, ref_, path.as_deref(), project_paths)?;
1424 let (commit, staging_path) = match &checkout {
1425 GitCheckout::InPlace { commit } => (commit, None),
1426 GitCheckout::Staged {
1427 commit,
1428 staging_path,
1429 } => (commit, Some(staging_path)),
1430 };
1431
1432 // Use the package's location in the staging worktree, where its gleam.toml
1433 // lives, not build/packages/. A `../sibling` path dep is resolved next to
1434 // that gleam.toml, and the sibling is only present in the worktree.
1435 let (package_path, repo_root) = match (&path, staging_path) {
1436 (Some(subdir), Some(staging_path)) => {
1437 let repo_root = fs::canonicalise(staging_path)?;
1438 (fs::canonicalise(&staging_path.join(subdir))?, repo_root)
1439 }
1440 _ => {
1441 let package_path =
1442 fs::canonicalise(&project_paths.build_packages_package(&package_name))?;
1443 (package_path.clone(), package_path)
1444 }
1445 };
1446
1447 let version = provide_package(
1448 package_name,
1449 package_path,
1450 SourceContext::Git {
1451 repo: repo.into(),
1452 commit: commit.clone(),
1453 path,
1454 repo_root: &repo_root,
1455 },
1456 project_paths,
1457 provided,
1458 parents,
1459 )?;
1460
1461 checkout.cleanup()?;
1462 Ok(version)
1463}
1464
1465/// Adds a gleam project located at a specific path to the list of "provided packages"
1466fn provide_package(
1467 package_name: EcoString,
1468 package_path: Utf8PathBuf,
1469 source: SourceContext<'_>,
1470 project_paths: &ProjectPaths,
1471 provided: &mut HashMap<EcoString, ProvidedPackage>,
1472 parents: &mut Vec<EcoString>,
1473) -> Result<hexpm::version::Range> {
1474 let package_source = source.to_provided_source();
1475
1476 // Return early if a package cycle is detected
1477 if parents.contains(&package_name) {
1478 let mut last_cycle = parents
1479 .split(|p| p == &package_name)
1480 .next_back()
1481 .unwrap_or_default()
1482 .to_vec();
1483 last_cycle.push(package_name);
1484 return Err(Error::PackageCycle {
1485 packages: last_cycle,
1486 });
1487 }
1488 // Check that we do not have a cached version of this package already
1489 match provided.get(&package_name) {
1490 Some(package) if package.source == package_source => {
1491 // This package has already been provided from this source, return the version
1492 let version = hexpm::version::Range::new(format!("== {}", package.version))
1493 .expect("== {version} should be a valid range");
1494 return Ok(version);
1495 }
1496 Some(package) => {
1497 // This package has already been provided from a different source which conflicts
1498 return Err(Error::ProvidedDependencyConflict {
1499 package: package_name.into(),
1500 source_1: package_source.to_toml(),
1501 source_2: package.source.to_toml(),
1502 });
1503 }
1504 None => (),
1505 }
1506 // Load the package
1507 let config = crate::config::read(package_path.join("gleam.toml"))?;
1508 // Check that we are loading the correct project
1509 if config.name != package_name {
1510 return Err(Error::WrongDependencyProvided {
1511 expected: package_name.into(),
1512 path: package_path.to_path_buf(),
1513 found: config.name.into(),
1514 });
1515 };
1516 // Walk the requirements of the package
1517 let mut requirements = HashMap::new();
1518 parents.push(package_name);
1519 for (name, requirement) in config.dependencies.into_iter() {
1520 let version = match requirement {
1521 Requirement::Hex { version } => version,
1522 Requirement::Path { path } => match &source {
1523 // A path dependency of a git package points to another
1524 // package within the same repository, so lock it as a git
1525 // source to keep the manifest portable.
1526 SourceContext::Git {
1527 repo,
1528 commit,
1529 repo_root,
1530 ..
1531 } => {
1532 let (child_path, child_repo_path) =
1533 resolve_git_path_package(&name, &path, repo, &package_path, repo_root)?;
1534 // A path dependency resolving to the repository root has an
1535 // empty repo-relative path. Normalise it to `None` so it
1536 // matches a package declared directly as a git dependency
1537 // with no path.
1538 let child_repo_path = if child_repo_path.as_str().is_empty() {
1539 None
1540 } else {
1541 Some(child_repo_path)
1542 };
1543
1544 provide_package(
1545 name.clone(),
1546 child_path,
1547 SourceContext::Git {
1548 repo: repo.clone(),
1549 commit: commit.clone(),
1550 path: child_repo_path,
1551 repo_root,
1552 },
1553 project_paths,
1554 provided,
1555 parents,
1556 )?
1557 }
1558 SourceContext::Local { .. } => {
1559 // Recursively walk local packages
1560 provide_local_package(
1561 name.clone(),
1562 &path,
1563 &package_path,
1564 project_paths,
1565 provided,
1566 parents,
1567 )?
1568 }
1569 },
1570 Requirement::Git { git, ref_, path } => provide_git_package(
1571 name.clone(),
1572 &git,
1573 &ref_,
1574 path,
1575 project_paths,
1576 provided,
1577 parents,
1578 )?,
1579 };
1580 let _ = requirements.insert(name, version);
1581 }
1582 let _ = parents.pop();
1583 // Add the package to the provided packages dictionary
1584 let version = hexpm::version::Range::new(format!("== {}", config.version))
1585 .expect("== {version} should be a valid range");
1586 let _ = provided.insert(
1587 config.name,
1588 ProvidedPackage {
1589 version: config.version,
1590 source: package_source,
1591 requirements,
1592 },
1593 );
1594 // Return the version
1595 Ok(version)
1596}
1597
1598/// Unlocks specified packages and their unique dependencies.
1599///
1600/// If a manifest is provided, it also unlocks indirect dependencies that are
1601/// not required by any other package or the root project.
1602pub fn unlock_packages(
1603 locked: &mut HashMap<EcoString, Version>,
1604 packages_to_unlock: &[EcoString],
1605 manifest: Option<&Manifest>,
1606) -> Result<()> {
1607 if let Some(manifest) = manifest {
1608 let mut packages_to_unlock: Vec<EcoString> = packages_to_unlock.to_vec();
1609
1610 while let Some(package_name) = packages_to_unlock.pop() {
1611 if locked.remove(&package_name).is_some()
1612 && let Some(package) = manifest.packages.iter().find(|p| p.name == package_name)
1613 {
1614 let deps_to_unlock = find_deps_to_unlock(package, locked, manifest);
1615 packages_to_unlock.extend(deps_to_unlock);
1616 }
1617 }
1618 } else {
1619 for package_name in packages_to_unlock {
1620 let _ = locked.remove(package_name);
1621 }
1622 }
1623
1624 Ok(())
1625}
1626
1627/// Identifies which dependencies of a package should be unlocked.
1628///
1629/// A dependency is eligible for unlocking if it is currently locked,
1630/// is not a root dependency, and is not required by any locked package.
1631fn find_deps_to_unlock(
1632 package: &ManifestPackage,
1633 locked: &HashMap<EcoString, Version>,
1634 manifest: &Manifest,
1635) -> Vec<EcoString> {
1636 package
1637 .requirements
1638 .iter()
1639 .filter(|&dep| {
1640 locked.contains_key(dep)
1641 && !manifest.requirements.contains_key(dep)
1642 && manifest
1643 .packages
1644 .iter()
1645 .all(|p| !locked.contains_key(&p.name) || !p.requirements.contains(dep))
1646 })
1647 .cloned()
1648 .collect()
1649}
1650
1651/// Determine the information to add to the manifest for a specific package
1652async fn lookup_package(
1653 name: String,
1654 version: Version,
1655 provided: &HashMap<EcoString, ProvidedPackage>,
1656 credentials: Option<&hexpm::Credentials>,
1657) -> Result<ManifestPackage> {
1658 match provided.get(name.as_str()) {
1659 Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
1660 None => {
1661 let config = hexpm::Config::new();
1662 let release =
1663 hex::get_package_release(&name, &version, credentials, &config, &HttpClient::new())
1664 .await?;
1665 let build_tools = release
1666 .meta
1667 .build_tools
1668 .iter()
1669 .map(|s| EcoString::from(s.as_str()))
1670 .collect_vec();
1671 let requirements = release
1672 .requirements
1673 .keys()
1674 .map(|s| EcoString::from(s.as_str()))
1675 .collect_vec();
1676 Ok(ManifestPackage {
1677 name: name.into(),
1678 version,
1679 otp_app: Some(release.meta.app.into()),
1680 build_tools,
1681 requirements,
1682 source: ManifestPackageSource::Hex {
1683 outer_checksum: Base16Checksum(release.outer_checksum),
1684 },
1685 })
1686 }
1687 }
1688}
1689
1690struct PackageFetcher {
1691 runtime_cache: RefCell<HashMap<String, Rc<hexpm::Package>>>,
1692 runtime: tokio::runtime::Handle,
1693 http: HttpClient,
1694 credentials: Option<hexpm::Credentials>,
1695}
1696
1697impl PackageFetcher {
1698 pub fn new(runtime: tokio::runtime::Handle) -> Self {
1699 Self {
1700 runtime_cache: RefCell::new(HashMap::new()),
1701 runtime,
1702 http: HttpClient::new(),
1703 credentials: crate::hex::read_env_readonly_api_key(),
1704 }
1705 }
1706
1707 /// Caches the result of `get_dependencies` so that we don't need to make a network request.
1708 /// Currently dependencies are fetched during initial version resolution, and then during check
1709 /// for major version availability.
1710 fn cache_package(&self, package: &str, result: Rc<hexpm::Package>) {
1711 let mut runtime_cache = self.runtime_cache.borrow_mut();
1712 let _ = runtime_cache.insert(package.to_string(), result);
1713 }
1714}
1715
1716#[derive(Debug)]
1717pub struct Untar;
1718
1719impl Untar {
1720 pub fn boxed() -> Box<Self> {
1721 Box::new(Self)
1722 }
1723}
1724
1725impl TarUnpacker for Untar {
1726 fn io_result_entries<'a>(
1727 &self,
1728 archive: &'a mut tar::Archive<WrappedReader>,
1729 ) -> std::io::Result<tar::Entries<'a, WrappedReader>> {
1730 archive.entries()
1731 }
1732
1733 fn io_result_unpack(
1734 &self,
1735 path: &Utf8Path,
1736 mut archive: tar::Archive<GzDecoder<tar::Entry<'_, WrappedReader>>>,
1737 ) -> std::io::Result<()> {
1738 archive.unpack(path)
1739 }
1740}
1741
1742impl dependency::PackageFetcher for PackageFetcher {
1743 fn get_dependencies(&self, package: &str) -> Result<Rc<hexpm::Package>, PackageFetchError> {
1744 {
1745 let runtime_cache = self.runtime_cache.borrow();
1746 let result = runtime_cache.get(package);
1747
1748 if let Some(result) = result {
1749 return Ok(result.clone());
1750 }
1751 }
1752
1753 tracing::debug!(package = package, "looking_up_hex_package");
1754 let config = hexpm::Config::new();
1755 let request =
1756 hexpm::repository_v2_get_package_request(package, self.credentials.as_ref(), &config);
1757 let response = self
1758 .runtime
1759 .block_on(self.http.send(request))
1760 .map_err(PackageFetchError::fetch_error)?;
1761
1762 let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
1763 .map_err(|e| PackageFetchError::from_api_error(e, package))?;
1764 let pkg = Rc::new(pkg);
1765 let pkg_ref = Rc::clone(&pkg);
1766 self.cache_package(package, pkg);
1767 Ok(pkg_ref)
1768 }
1769}