nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
44 kB
1324 lines
1mod dependency_manager;
2
3use std::{
4 cell::RefCell,
5 collections::{HashMap, HashSet},
6 io::ErrorKind,
7 process::Command,
8 rc::Rc,
9 time::Instant,
10};
11
12use camino::{Utf8Path, Utf8PathBuf};
13use ecow::{EcoString, eco_format};
14use flate2::read::GzDecoder;
15use gleam_core::{
16 Error, Result,
17 build::{Mode, SourceFingerprint, Target, Telemetry},
18 config::PackageConfig,
19 dependency::{self, PackageFetchError},
20 error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
21 hex::{self, HEXPM_PUBLIC_KEY},
22 io::{HttpClient as _, TarUnpacker, WrappedReader},
23 manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, PackageChanges},
24 paths::ProjectPaths,
25 requirement::Requirement,
26};
27use hexpm::version::Version;
28use itertools::Itertools;
29use same_file::is_same_file;
30use strum::IntoEnumIterator;
31
32pub use dependency_manager::DependencyManagerConfig;
33
34#[cfg(test)]
35mod tests;
36
37use crate::{
38 TreeOptions,
39 build_lock::{BuildLock, Guard},
40 cli,
41 fs::{self, ProjectIO},
42 http::HttpClient,
43 text_layout::space_table,
44};
45
46struct Symbols {
47 down: &'static str,
48 tee: &'static str,
49 ell: &'static str,
50 right: &'static str,
51}
52
53static UTF8_SYMBOLS: Symbols = Symbols {
54 down: "│",
55 tee: "├",
56 ell: "└",
57 right: "─",
58};
59
60/// When set to `Yes`, the cli will check for major version updates of direct dependencies and
61/// print them to the console if the major versions are not upgradeable due to constraints.
62#[derive(Debug, Clone, Copy)]
63pub enum CheckMajorVersions {
64 Yes,
65 No,
66}
67
68pub fn list(paths: &ProjectPaths) -> Result<()> {
69 let (_, manifest) = get_manifest_details(paths)?;
70 list_manifest_packages(std::io::stdout(), manifest)
71}
72
73pub fn tree(paths: &ProjectPaths, options: TreeOptions) -> Result<()> {
74 let (config, manifest) = get_manifest_details(paths)?;
75
76 // Initialize the root package since it is not part of the manifest
77 let root_package = ManifestPackage {
78 build_tools: vec![],
79 name: config.name.clone(),
80 requirements: config.all_direct_dependencies()?.keys().cloned().collect(),
81 version: config.version.clone(),
82 source: ManifestPackageSource::Local {
83 path: paths.root().to_path_buf(),
84 },
85 otp_app: None,
86 };
87
88 // Get the manifest packages and add the root package to the vec
89 let mut packages = manifest.packages.iter().cloned().collect_vec();
90 packages.append(&mut vec![root_package.clone()]);
91
92 list_package_and_dependencies_tree(std::io::stdout(), options, packages.clone(), config.name)
93}
94
95fn get_manifest_details(paths: &ProjectPaths) -> Result<(PackageConfig, Manifest)> {
96 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
97 let config = crate::config::root_config(paths)?;
98 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
99 let dependency_manager = DependencyManagerConfig {
100 use_manifest: UseManifest::Yes,
101 check_major_versions: CheckMajorVersions::No,
102 }
103 .into_dependency_manager(
104 runtime.handle().clone(),
105 package_fetcher,
106 cli::Reporter::new(),
107 Mode::Dev,
108 );
109 let manifest = dependency_manager
110 .resolve_versions(paths, &config, Vec::new())?
111 .manifest;
112 Ok((config, manifest))
113}
114
115fn list_manifest_packages<W: std::io::Write>(mut buffer: W, manifest: Manifest) -> Result<()> {
116 let packages = manifest
117 .packages
118 .into_iter()
119 .map(|package| vec![package.name.to_string(), package.version.to_string()])
120 .collect_vec();
121 let out = space_table(&["Package", "Version"], packages);
122
123 write!(buffer, "{out}").map_err(|e| Error::StandardIo {
124 action: StandardIoAction::Write,
125 err: Some(e.kind()),
126 })
127}
128
129fn list_package_and_dependencies_tree<W: std::io::Write>(
130 mut buffer: W,
131 options: TreeOptions,
132 packages: Vec<ManifestPackage>,
133 root_package_name: EcoString,
134) -> Result<()> {
135 let mut invert = false;
136
137 let package: Option<&ManifestPackage> = if let Some(input_package_name) = options.package {
138 packages.iter().find(|p| p.name == input_package_name)
139 } else if let Some(input_package_name) = options.invert {
140 invert = true;
141 packages.iter().find(|p| p.name == input_package_name)
142 } else {
143 packages.iter().find(|p| p.name == root_package_name)
144 };
145
146 if let Some(package) = package {
147 let tree = Vec::from([eco_format!("{0} v{1}", package.name, package.version)]);
148 let tree = list_dependencies_tree(
149 tree.clone(),
150 package.clone(),
151 packages,
152 EcoString::new(),
153 invert,
154 );
155
156 tree.iter()
157 .try_for_each(|line| writeln!(buffer, "{line}"))
158 .map_err(|e| Error::StandardIo {
159 action: StandardIoAction::Write,
160 err: Some(e.kind()),
161 })
162 } else {
163 writeln!(buffer, "Package not found. Please check the package name.").map_err(|e| {
164 Error::StandardIo {
165 action: StandardIoAction::Write,
166 err: Some(e.kind()),
167 }
168 })
169 }
170}
171
172fn list_dependencies_tree(
173 mut tree: Vec<EcoString>,
174 package: ManifestPackage,
175 packages: Vec<ManifestPackage>,
176 accum: EcoString,
177 invert: bool,
178) -> Vec<EcoString> {
179 let dependencies = packages
180 .iter()
181 .filter(|p| {
182 (invert && p.requirements.contains(&package.name))
183 || (!invert && package.requirements.contains(&p.name))
184 })
185 .cloned()
186 .collect_vec();
187
188 let dependencies = dependencies.iter().sorted().enumerate();
189
190 let deps_length = dependencies.len();
191 for (index, dependency) in dependencies {
192 let is_last = index == deps_length - 1;
193 let prefix = if is_last {
194 UTF8_SYMBOLS.ell
195 } else {
196 UTF8_SYMBOLS.tee
197 };
198
199 tree.push(eco_format!(
200 "{0}{1}{2}{2} {3} v{4}",
201 accum.clone(),
202 prefix,
203 UTF8_SYMBOLS.right,
204 dependency.name,
205 dependency.version
206 ));
207
208 let accum = accum.clone() + (if !is_last { UTF8_SYMBOLS.down } else { " " }) + " ";
209
210 tree = list_dependencies_tree(
211 tree.clone(),
212 dependency.clone(),
213 packages.clone(),
214 accum.clone(),
215 invert,
216 );
217 }
218
219 tree
220}
221
222pub fn outdated(paths: &ProjectPaths) -> Result<()> {
223 let (_, manifest) = get_manifest_details(paths)?;
224
225 let total_packages = manifest
226 .packages
227 .iter()
228 .filter(|package| matches!(package.source, ManifestPackageSource::Hex { .. }))
229 .count();
230
231 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
232 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
233
234 let version_updates = dependency::check_for_version_updates(&manifest, &package_fetcher);
235
236 print!(
237 "{}",
238 pretty_print_outdated_versions(total_packages, version_updates)
239 );
240
241 Ok(())
242}
243
244#[derive(Debug, Clone, Copy)]
245pub enum UseManifest {
246 Yes,
247 No,
248}
249
250pub fn update(paths: &ProjectPaths, packages: Vec<String>) -> Result<()> {
251 let use_manifest = if packages.is_empty() {
252 UseManifest::No
253 } else {
254 UseManifest::Yes
255 };
256
257 // Update specific packages
258 _ = resolve_and_download(
259 paths,
260 cli::Reporter::new(),
261 None,
262 packages.into_iter().map(EcoString::from).collect(),
263 DependencyManagerConfig {
264 use_manifest,
265 check_major_versions: CheckMajorVersions::Yes,
266 },
267 )?;
268
269 Ok(())
270}
271
272/// Edit the manifest.toml file in this proejct, removing all extra requirements and packages
273/// that are no longer present in the gleam.toml config.
274pub fn cleanup<Telem: Telemetry>(paths: &ProjectPaths, telemetry: Telem) -> Result<Manifest> {
275 let span = tracing::info_span!("remove_deps");
276 let _enter = span.enter();
277
278 // We do this before acquiring the build lock so that we don't create the
279 // build directory if there is no gleam.toml
280 crate::config::ensure_config_exists(paths)?;
281
282 let lock = BuildLock::new_packages(paths)?;
283 let _guard: Guard = lock.lock(&telemetry)?;
284
285 // Read the project config
286 let config = crate::root_config(paths)?;
287 let old_manifest = read_manifest_from_disc(paths)?;
288 let mut manifest = old_manifest.clone();
289
290 remove_extra_requirements(&config, &mut manifest)?;
291
292 // Remove any packages that are no longer required due to manifest changes
293 let local = LocalPackages::read_from_disc(paths)?;
294 remove_extra_packages(paths, &local, &manifest, &telemetry)?;
295
296 // Record new state of the packages directory
297 tracing::debug!("writing_manifest_toml");
298 write_manifest_to_disc(paths, &manifest)?;
299 LocalPackages::from_manifest(&manifest).write_to_disc(paths)?;
300
301 let changes = PackageChanges::between_manifests(&old_manifest, &manifest);
302 telemetry.resolved_package_versions(&changes);
303
304 Ok(manifest)
305}
306
307/// Remove requirements and unneeded packages from manifest that are no longer present in config.
308fn remove_extra_requirements(config: &PackageConfig, manifest: &mut Manifest) -> Result<()> {
309 // "extra requirements" are all packages that are requirements in the manifest, but no longer
310 // part of the gleam.toml config.
311 let is_extra_requirement = |name: &EcoString| {
312 !config.dev_dependencies.contains_key(name) && !config.dependencies.contains_key(name)
313 };
314
315 // If a requirement is also used as a dependency, we do not want to force-unlock it.
316 // If the dependents get deleted as well, this transitive dependency will be dropped.
317 let is_unlockable_requirement = |name: &EcoString| {
318 manifest
319 .packages
320 .iter()
321 .all(|p| !p.requirements.contains(name))
322 };
323
324 let extra_requirements = manifest
325 .requirements
326 .keys()
327 .filter(|&name| is_extra_requirement(name) && is_unlockable_requirement(name))
328 .cloned()
329 .collect::<Vec<_>>();
330
331 manifest
332 .requirements
333 .retain(|name, _| !is_extra_requirement(name));
334
335 // Unlock all packages that we we want to remove - this removes them and all unneeded
336 // dependencies from `locked`.
337 let mut locked = config.locked(Some(manifest))?;
338 unlock_packages(&mut locked, extra_requirements.as_slice(), Some(manifest))?;
339 // Remove all unlocked packages from the manifest - these are truly no longer needed.
340 manifest
341 .packages
342 .retain(|package| locked.contains_key(&package.name));
343
344 Ok(())
345}
346
347pub fn parse_gleam_add_specifier(package: &str) -> Result<(EcoString, Requirement)> {
348 let Some((package, version)) = package.split_once('@') else {
349 // Default to the latest version available.
350 return Ok((
351 package.into(),
352 Requirement::hex(">= 0.0.0").expect("'>= 0.0.0' should be a valid pubgrub range"),
353 ));
354 };
355
356 // Parse the major and minor from the provided semantic version.
357 let parts = version.split('.').collect::<Vec<_>>();
358 let major = match parts.first() {
359 Some(major) => Ok(major),
360 None => Err(Error::InvalidVersionFormat {
361 input: package.to_string(),
362 error: "Failed to parse semantic major version".to_string(),
363 }),
364 }?;
365 let minor = match parts.get(1) {
366 Some(minor) => minor,
367 None => "0",
368 };
369
370 // Using the major version specifier, calculate the maximum
371 // allowable version (i.e., the next major version).
372 let Ok(num) = major.parse::<usize>() else {
373 return Err(Error::InvalidVersionFormat {
374 input: version.to_string(),
375 error: "Failed to parse semantic major version as integer".to_string(),
376 });
377 };
378
379 let max_ver = [&(num + 1).to_string(), "0", "0"].join(".");
380
381 // Pad the provided version specifier with zeros map to a Hex version.
382 let requirement = match parts.len() {
383 1 | 2 => {
384 let min_ver = [major, minor, "0"].join(".");
385 Requirement::hex(&[">=", &min_ver, "and", "<", &max_ver].join(" "))
386 }
387 3 => Requirement::hex(version),
388 n => {
389 return Err(Error::InvalidVersionFormat {
390 input: version.to_string(),
391 error: format!(
392 "Expected up to 3 numbers in version specifier (MAJOR.MINOR.PATCH), found {n}"
393 ),
394 });
395 }
396 }?;
397
398 Ok((package.into(), requirement))
399}
400
401pub fn resolve_and_download<Telem: Telemetry>(
402 paths: &ProjectPaths,
403 telemetry: Telem,
404 new_package: Option<(Vec<(EcoString, Requirement)>, bool)>,
405 packages_to_update: Vec<EcoString>,
406 config: DependencyManagerConfig,
407) -> Result<Manifest> {
408 // Start event loop so we can run async functions to call the Hex API
409 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
410 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
411
412 let dependency_manager = config.into_dependency_manager(
413 runtime.handle().clone(),
414 package_fetcher,
415 telemetry,
416 Mode::Dev,
417 );
418
419 dependency_manager.resolve_and_download_versions(paths, new_package, packages_to_update)
420}
421
422fn format_versions_and_extract_longest_parts(
423 versions: dependency::PackageVersionDiffs,
424) -> Vec<Vec<String>> {
425 versions
426 .iter()
427 .map(|(name, (v1, v2))| vec![name.to_string(), v1.to_string(), v2.to_string()])
428 .sorted()
429 .collect_vec()
430}
431
432fn pretty_print_major_versions_available(versions: dependency::PackageVersionDiffs) -> String {
433 let versions = format_versions_and_extract_longest_parts(versions);
434
435 format!(
436 "\nThe following dependencies have new major versions available:\n\n{}",
437 space_table(&["Package", "Current", "Latest"], &versions)
438 )
439}
440
441fn pretty_print_version_updates(versions: dependency::PackageVersionDiffs) -> EcoString {
442 let versions = format_versions_and_extract_longest_parts(versions);
443 space_table(&["Package", "Current", "Latest"], &versions)
444}
445
446fn pretty_print_outdated_versions(
447 total_packages: usize,
448 versions: dependency::PackageVersionDiffs,
449) -> EcoString {
450 let summary = eco_format!(
451 "{} of {} packages have newer versions available.",
452 versions.len(),
453 total_packages
454 );
455
456 if versions.is_empty() {
457 eco_format!("{}\n", summary)
458 } else {
459 eco_format!("{}\n\n{}", summary, pretty_print_version_updates(versions))
460 }
461}
462
463async fn add_missing_packages<Telem: Telemetry>(
464 paths: &ProjectPaths,
465 fs: Box<ProjectIO>,
466 manifest: &Manifest,
467 local: &LocalPackages,
468 project_name: EcoString,
469 telemetry: &Telem,
470) -> Result<(), Error> {
471 let missing_packages = local.missing_local_packages(manifest, &project_name);
472
473 let mut num_to_download = 0;
474
475 let missing_git_packages = missing_packages
476 .iter()
477 .copied()
478 .filter(|package| package.is_git())
479 .inspect(|_| {
480 num_to_download += 1;
481 })
482 .collect_vec();
483
484 let mut missing_hex_packages = missing_packages
485 .iter()
486 .copied()
487 .filter(|package| package.is_hex())
488 .inspect(|_| {
489 num_to_download += 1;
490 })
491 .peekable();
492
493 // If we need to download at-least one package
494 if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() {
495 let http = HttpClient::boxed();
496 let downloader = hex::Downloader::new(fs.clone(), fs, http, Untar::boxed(), paths.clone());
497 let start = Instant::now();
498 telemetry.downloading_package("packages");
499 downloader
500 .download_hex_packages(missing_hex_packages, &project_name)
501 .await?;
502 for package in missing_git_packages {
503 let ManifestPackageSource::Git { repo, commit } = &package.source else {
504 continue;
505 };
506 let _ = download_git_package(&package.name, repo, commit, paths)?;
507 }
508 telemetry.packages_downloaded(start, num_to_download);
509 }
510
511 Ok(())
512}
513
514fn remove_extra_packages<Telem: Telemetry>(
515 paths: &ProjectPaths,
516 local: &LocalPackages,
517 manifest: &Manifest,
518 telemetry: &Telem,
519) -> Result<()> {
520 let _guard = BuildLock::lock_all_build(paths, telemetry)?;
521
522 for (package_name, version) in local.extra_local_packages(manifest) {
523 // TODO: test
524 // Delete the package source
525 let path = paths.build_packages_package(&package_name);
526 if path.exists() {
527 tracing::debug!(package=%package_name, version=%version, "removing_unneeded_package");
528 fs::delete_directory(&path)?;
529 }
530
531 // TODO: test
532 // Delete any build artefacts for the package
533 for mode in Mode::iter() {
534 for target in Target::iter() {
535 let name = manifest
536 .packages
537 .iter()
538 .find(|p| p.name == package_name)
539 .map(|p| p.application_name().as_str())
540 .unwrap_or(package_name.as_str());
541 let path = paths.build_directory_for_package(mode, target, name);
542 if path.exists() {
543 tracing::debug!(package=%package_name, version=%version, "deleting_build_cache");
544 fs::delete_directory(&path)?;
545 }
546 }
547 }
548 }
549 Ok(())
550}
551
552fn read_manifest_from_disc(paths: &ProjectPaths) -> Result<Manifest> {
553 tracing::debug!("reading_manifest_toml");
554 let manifest_path = paths.manifest();
555 let toml = fs::read(&manifest_path)?;
556 let manifest = toml::from_str(&toml).map_err(|e| Error::FileIo {
557 action: FileIoAction::Parse,
558 kind: FileKind::File,
559 path: manifest_path.clone(),
560 err: Some(e.to_string()),
561 })?;
562 Ok(manifest)
563}
564
565fn write_manifest_to_disc(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
566 let path = paths.manifest();
567 fs::write(&path, &manifest.to_toml(paths.root()))
568}
569
570// This is the container for locally pinned packages, representing the current contents of
571// the `project/build/packages` directory.
572// For descriptions of packages provided by paths and git deps, see the ProvidedPackage struct.
573// The same package may appear in both at different times.
574#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
575struct LocalPackages {
576 #[serde(deserialize_with = "gleam_core::config::map_with_package_name_keys::deserialize")]
577 packages: HashMap<EcoString, Version>,
578}
579
580impl LocalPackages {
581 pub fn extra_local_packages(&self, manifest: &Manifest) -> Vec<(EcoString, Version)> {
582 let manifest_packages: HashSet<_> = manifest
583 .packages
584 .iter()
585 .map(|p| (&p.name, &p.version))
586 .collect();
587 self.packages
588 .iter()
589 .filter(|(n, v)| !manifest_packages.contains(&(&EcoString::from(*n), v)))
590 .map(|(n, v)| (n.clone(), v.clone()))
591 .collect()
592 }
593
594 pub fn missing_local_packages<'a>(
595 &self,
596 manifest: &'a Manifest,
597 root: &str,
598 ) -> Vec<&'a ManifestPackage> {
599 manifest
600 .packages
601 .iter()
602 // We don't need to download the root package
603 .filter(|p| p.name != root)
604 // We don't need to download local packages because we use the linked source directly
605 .filter(|p| !p.is_local())
606 // We don't need to download packages which we have the correct version of
607 .filter(|p| self.packages.get(p.name.as_str()) != Some(&p.version))
608 .collect()
609 }
610
611 pub fn read_from_disc(paths: &ProjectPaths) -> Result<Self> {
612 let path = paths.build_packages_toml();
613 if !path.exists() {
614 return Ok(Self {
615 packages: HashMap::new(),
616 });
617 }
618 let toml = fs::read(&path)?;
619 toml::from_str(&toml).map_err(|e| Error::FileIo {
620 action: FileIoAction::Parse,
621 kind: FileKind::File,
622 path: path.clone(),
623 err: Some(e.to_string()),
624 })
625 }
626
627 pub fn write_to_disc(&self, paths: &ProjectPaths) -> Result<()> {
628 let path = paths.build_packages_toml();
629 let toml = toml::to_string(&self).expect("packages.toml serialization");
630 fs::write(&path, &toml)
631 }
632
633 pub fn from_manifest(manifest: &Manifest) -> Self {
634 Self {
635 packages: manifest
636 .packages
637 .iter()
638 .map(|p| (p.name.clone(), p.version.clone()))
639 .collect(),
640 }
641 }
642}
643
644#[test]
645fn local_packages_deserialise_ok() {
646 let toml = r#"
647[packages]
648gleam_stdlib = "1.0.0"
649gleam_otp = "1.1.0"
650"#;
651 let packages: LocalPackages = toml::from_str(toml).unwrap();
652 assert_eq!(
653 packages,
654 LocalPackages {
655 packages: HashMap::from_iter([
656 ("gleam_stdlib".into(), Version::new(1, 0, 0)),
657 ("gleam_otp".into(), Version::new(1, 1, 0)),
658 ])
659 }
660 )
661}
662
663#[test]
664fn local_packages_deserialise_invalid_name() {
665 let toml = r#"
666[packages]
667gleam_stdlib = "1.0.0"
668"../../stuff" = "1.1.0"
669"#;
670 let error = toml::from_str::<LocalPackages>(toml)
671 .expect_err("should fail to deserialise because of invalid name");
672 insta::assert_snapshot!(insta::internals::AutoName, error.to_string());
673}
674
675fn is_same_requirements(
676 requirements1: &HashMap<EcoString, Requirement>,
677 requirements2: &HashMap<EcoString, Requirement>,
678 root_path: &Utf8Path,
679) -> Result<bool> {
680 if requirements1.len() != requirements2.len() {
681 return Ok(false);
682 }
683
684 for (key, requirement1) in requirements1 {
685 if !same_requirements(requirement1, requirements2.get(key), root_path)? {
686 return Ok(false);
687 }
688 }
689
690 Ok(true)
691}
692
693/// Returns true if all path dependency configs are unchanged since last build.
694///
695/// If any of the path dependency configs have changed that means that we need to
696/// re-perform dependency resolution, as their dependencies could have changed
697/// themselves.
698///
699/// We use gleam.toml rather than manifest.toml as:
700///
701/// 1. The dependency requirements could have changed but resolution not have been
702/// run in that package yet, so the manifest would be the same, resulting in us
703/// failing to detect that resolution is required.
704///
705/// 2. Dependency manifests are not used in any way, so a change in the manifest
706/// may not have any impact on this package.
707///
708/// This does mean that changes unrelated to the path dependency's dependencies
709/// will trigger resolution, but gleam.toml is edited rarely, and no-change
710/// resolution is fast enough, so that's OK.
711///
712/// Note: This does not check path dependencies of path dependencies! Changes to
713/// their configs will fail to be picked up. To resolve this we would need to keep
714/// a list of all the path dependencies in the project, instead of only the direct
715/// path dependencies.
716///
717fn path_dependency_configs_unchanged(
718 requirements: &HashMap<EcoString, Requirement>,
719 paths: &ProjectPaths,
720) -> Result<bool> {
721 for (name, requirement) in requirements {
722 let Requirement::Path { path } = requirement else {
723 continue;
724 };
725
726 let config_path = paths.path_dependency_gleam_toml_path(path);
727 let fingerprint_path = paths.dependency_gleam_toml_fingerprint_path(name.as_str());
728
729 // Check mtimes before hashing, to avoid extra work
730 if fingerprint_path.exists() {
731 let config_time = fs::modification_time(&config_path)?;
732 let fingerprint_time = fs::modification_time(&fingerprint_path)?;
733 if config_time <= fingerprint_time {
734 continue;
735 }
736 };
737
738 let config_text = fs::read(&config_path)?;
739 let current_fingerprint = SourceFingerprint::new(&config_text).to_numerical_string();
740
741 // If cached hash file doesn't exist, this is the first time we're checking this dependency
742 if !fingerprint_path.exists() {
743 // Save the current hash for future comparisons
744 fs::write(&fingerprint_path, ¤t_fingerprint)?;
745 return Ok(false);
746 }
747
748 let previous_fingerprint = fs::read(&fingerprint_path)?;
749
750 if previous_fingerprint != current_fingerprint {
751 tracing::debug!("path_dependency_config_changed_forcing_rebuild");
752 fs::write(&fingerprint_path, ¤t_fingerprint)?;
753 return Ok(false);
754 }
755 }
756
757 Ok(true)
758}
759
760fn same_requirements(
761 requirement1: &Requirement,
762 requirement2: Option<&Requirement>,
763 root_path: &Utf8Path,
764) -> Result<bool> {
765 let (left, right) = match (requirement1, requirement2) {
766 (Requirement::Path { path: path1 }, Some(Requirement::Path { path: path2 })) => {
767 let left = fs::canonicalise(&root_path.join(path1))?;
768 let right = fs::canonicalise(&root_path.join(path2))?;
769 (left, right)
770 }
771 (_, Some(requirement2)) => return Ok(requirement1 == requirement2),
772 (_, None) => return Ok(false),
773 };
774
775 Ok(left == right)
776}
777
778#[derive(Clone, Eq, PartialEq, Debug)]
779struct ProvidedPackage {
780 version: Version,
781 source: ProvidedPackageSource,
782 requirements: HashMap<EcoString, hexpm::version::Range>,
783}
784
785#[derive(Clone, Eq, Debug)]
786enum ProvidedPackageSource {
787 Git { repo: EcoString, commit: EcoString },
788 Local { path: Utf8PathBuf },
789}
790
791impl ProvidedPackage {
792 fn to_hex_package(&self, name: &EcoString) -> hexpm::Package {
793 let requirements = self
794 .requirements
795 .iter()
796 .map(|(name, version)| {
797 (
798 name.as_str().into(),
799 hexpm::Dependency {
800 requirement: version.clone(),
801 optional: false,
802 app: None,
803 repository: None,
804 },
805 )
806 })
807 .collect();
808 let release = hexpm::Release {
809 version: self.version.clone(),
810 requirements,
811 retirement_status: None,
812 outer_checksum: vec![],
813 meta: (),
814 };
815 hexpm::Package {
816 name: name.as_str().into(),
817 repository: "local".into(),
818 releases: vec![release],
819 }
820 }
821
822 fn to_manifest_package(&self, name: &str) -> ManifestPackage {
823 let mut package = ManifestPackage {
824 name: name.into(),
825 version: self.version.clone(),
826 otp_app: None, // Note, this will probably need to be set to something eventually
827 build_tools: vec!["gleam".into()],
828 requirements: self.requirements.keys().cloned().collect(),
829 source: self.source.to_manifest_package_source(),
830 };
831 package.requirements.sort();
832 package
833 }
834}
835
836impl ProvidedPackageSource {
837 fn to_manifest_package_source(&self) -> ManifestPackageSource {
838 match self {
839 Self::Git { repo, commit } => ManifestPackageSource::Git {
840 repo: repo.clone(),
841 commit: commit.clone(),
842 },
843 Self::Local { path } => ManifestPackageSource::Local { path: path.clone() },
844 }
845 }
846
847 fn to_toml(&self) -> String {
848 match self {
849 Self::Git { repo, commit } => {
850 format!(r#"{{ repo: "{repo}", commit: "{commit}" }}"#)
851 }
852 Self::Local { path } => {
853 format!(r#"{{ path: "{path}" }}"#)
854 }
855 }
856 }
857}
858
859impl PartialEq for ProvidedPackageSource {
860 fn eq(&self, other: &Self) -> bool {
861 match (self, other) {
862 (Self::Local { path: own_path }, Self::Local { path: other_path }) => {
863 is_same_file(own_path, other_path).unwrap_or(false)
864 }
865
866 (
867 Self::Git {
868 repo: own_repo,
869 commit: own_commit,
870 },
871 Self::Git {
872 repo: other_repo,
873 commit: other_commit,
874 },
875 ) => own_repo == other_repo && own_commit == other_commit,
876
877 (Self::Git { .. }, Self::Local { .. }) | (Self::Local { .. }, Self::Git { .. }) => {
878 false
879 }
880 }
881 }
882}
883
884/// Provide a package from a local project
885fn provide_local_package(
886 package_name: EcoString,
887 package_path: &Utf8Path,
888 parent_path: &Utf8Path,
889 project_paths: &ProjectPaths,
890 provided: &mut HashMap<EcoString, ProvidedPackage>,
891 parents: &mut Vec<EcoString>,
892) -> Result<hexpm::version::Range> {
893 let package_path = if package_path.is_absolute() {
894 package_path.to_path_buf()
895 } else {
896 fs::canonicalise(&parent_path.join(package_path))?
897 };
898
899 let package_source = ProvidedPackageSource::Local {
900 path: package_path.clone(),
901 };
902 provide_package(
903 package_name,
904 package_path,
905 package_source,
906 project_paths,
907 provided,
908 parents,
909 )
910}
911
912fn execute_command(command: &mut Command) -> Result<std::process::Output> {
913 let result = command.output();
914 match result {
915 Ok(output) if output.status.success() => Ok(output),
916 Ok(output) => {
917 let reason = match String::from_utf8(output.stderr) {
918 Ok(stderr) => ShellCommandFailureReason::ShellCommandError(stderr),
919 Err(_) => ShellCommandFailureReason::Unknown,
920 };
921 Err(Error::ShellCommand {
922 program: "git".into(),
923 reason,
924 })
925 }
926 Err(error) => Err(match error.kind() {
927 ErrorKind::NotFound => Error::ShellProgramNotFound {
928 program: "git".into(),
929 os: fs::get_os(),
930 },
931
932 other => Error::ShellCommand {
933 program: "git".into(),
934 reason: ShellCommandFailureReason::IoError(other),
935 },
936 }),
937 }
938}
939
940/// Downloads a git package from a remote repository. The commands that are run
941/// looks like this:
942///
943/// ```sh
944/// git init
945/// git remote remove origin
946/// git remote add origin <repo>
947/// git fetch origin
948/// git checkout <ref>
949/// git rev-parse HEAD
950/// ```
951///
952/// This is somewhat inefficient as we have to fetch the entire git history before
953/// switching to the exact commit we want. There a few alternatives to this:
954///
955/// - `git clone --depth 1 --branch="<ref>"` This works, but only allows us to use
956/// branch names as refs, however we want to allow commit hashes as well.
957/// - `git fetch --depth 1 origin <ref>` Similarly, this imposes an unwanted
958/// restriction. `git fetch` only allows branch names or full commit hashes,
959/// but we want to allow partial hashes as well.
960///
961/// Since Git dependencies will be used quite rarely, this option was settled upon
962/// because it allows branch names, full and partial commit hashes as refs.
963///
964/// In the future we can optimise this more, for example first checking if we
965/// are already checked out to the commit stored in the manifest, or by only
966/// fetching the history without the objects to resolve partial commit hashes.
967/// For now though this is good enough until it become an actual performance
968/// problem.
969///
970fn download_git_package(
971 package_name: &str,
972 repo: &str,
973 ref_: &str,
974 project_paths: &ProjectPaths,
975) -> Result<EcoString> {
976 let package_path = project_paths.build_packages_package(package_name);
977
978 // If the package path exists but is not inside a git work tree, we need to
979 // remove the directory because running `git init` in a non-empty directory
980 // followed by `git checkout ...` is an error. See
981 // https://github.com/gleam-lang/gleam/issues/4488 for details.
982 if !fs::is_git_work_tree_root(&package_path) {
983 fs::delete_directory(&package_path)?;
984 }
985
986 fs::mkdir(&package_path)?;
987
988 let _ = execute_command(Command::new("git").arg("init").current_dir(&package_path))?;
989
990 // If this directory already exists, but the remote URL has been edited in
991 // `gleam.toml` without a `gleam clean`, `git remote add` will fail, causing
992 // the remote to be stuck as the original value. Here we remove the remote
993 // first, which ensures that `git remote add` properly add the remote each
994 // time. If this fails, that means we haven't set the remote in the first
995 // place, so we can safely ignore the error.
996 let _ = Command::new("git")
997 .arg("remote")
998 .arg("remove")
999 .arg("origin")
1000 .current_dir(&package_path)
1001 .output();
1002
1003 let _ = execute_command(
1004 Command::new("git")
1005 .arg("remote")
1006 .arg("add")
1007 .arg("origin")
1008 .arg(repo)
1009 .current_dir(&package_path),
1010 )?;
1011
1012 let _ = execute_command(
1013 Command::new("git")
1014 .arg("fetch")
1015 .arg("origin")
1016 .current_dir(&package_path),
1017 )?;
1018
1019 let _ = execute_command(
1020 Command::new("git")
1021 .arg("checkout")
1022 .arg(ref_)
1023 .current_dir(&package_path),
1024 )?;
1025
1026 let output = execute_command(
1027 Command::new("git")
1028 .arg("rev-parse")
1029 .arg("HEAD")
1030 .current_dir(&package_path),
1031 )?;
1032
1033 let commit = String::from_utf8(output.stdout)
1034 .expect("Output should be UTF-8")
1035 .trim()
1036 .into();
1037
1038 Ok(commit)
1039}
1040
1041/// Provide a package from a git repository
1042fn provide_git_package(
1043 package_name: EcoString,
1044 repo: &str,
1045 // A git ref, such as a branch name, commit hash or tag name
1046 ref_: &str,
1047 project_paths: &ProjectPaths,
1048 provided: &mut HashMap<EcoString, ProvidedPackage>,
1049 parents: &mut Vec<EcoString>,
1050) -> Result<hexpm::version::Range> {
1051 let commit = download_git_package(&package_name, repo, ref_, project_paths)?;
1052
1053 let package_source = ProvidedPackageSource::Git {
1054 repo: repo.into(),
1055 commit,
1056 };
1057
1058 let package_path = fs::canonicalise(&project_paths.build_packages_package(&package_name))?;
1059
1060 provide_package(
1061 package_name,
1062 package_path,
1063 package_source,
1064 project_paths,
1065 provided,
1066 parents,
1067 )
1068}
1069
1070/// Adds a gleam project located at a specific path to the list of "provided packages"
1071fn provide_package(
1072 package_name: EcoString,
1073 package_path: Utf8PathBuf,
1074 package_source: ProvidedPackageSource,
1075 project_paths: &ProjectPaths,
1076 provided: &mut HashMap<EcoString, ProvidedPackage>,
1077 parents: &mut Vec<EcoString>,
1078) -> Result<hexpm::version::Range> {
1079 // Return early if a package cycle is detected
1080 if parents.contains(&package_name) {
1081 let mut last_cycle = parents
1082 .split(|p| p == &package_name)
1083 .next_back()
1084 .unwrap_or_default()
1085 .to_vec();
1086 last_cycle.push(package_name);
1087 return Err(Error::PackageCycle {
1088 packages: last_cycle,
1089 });
1090 }
1091 // Check that we do not have a cached version of this package already
1092 match provided.get(&package_name) {
1093 Some(package) if package.source == package_source => {
1094 // This package has already been provided from this source, return the version
1095 let version = hexpm::version::Range::new(format!("== {}", &package.version))
1096 .expect("== {version} should be a valid range");
1097 return Ok(version);
1098 }
1099 Some(package) => {
1100 // This package has already been provided from a different source which conflicts
1101 return Err(Error::ProvidedDependencyConflict {
1102 package: package_name.into(),
1103 source_1: package_source.to_toml(),
1104 source_2: package.source.to_toml(),
1105 });
1106 }
1107 None => (),
1108 }
1109 // Load the package
1110 let config = crate::config::read(package_path.join("gleam.toml"))?;
1111 // Check that we are loading the correct project
1112 if config.name != package_name {
1113 return Err(Error::WrongDependencyProvided {
1114 expected: package_name.into(),
1115 path: package_path.to_path_buf(),
1116 found: config.name.into(),
1117 });
1118 };
1119 // Walk the requirements of the package
1120 let mut requirements = HashMap::new();
1121 parents.push(package_name);
1122 for (name, requirement) in config.dependencies.into_iter() {
1123 let version = match requirement {
1124 Requirement::Hex { version } => version,
1125 Requirement::Path { path } => {
1126 // Recursively walk local packages
1127 provide_local_package(
1128 name.clone(),
1129 &path,
1130 &package_path,
1131 project_paths,
1132 provided,
1133 parents,
1134 )?
1135 }
1136 Requirement::Git { git, ref_ } => {
1137 provide_git_package(name.clone(), &git, &ref_, project_paths, provided, parents)?
1138 }
1139 };
1140 let _ = requirements.insert(name, version);
1141 }
1142 let _ = parents.pop();
1143 // Add the package to the provided packages dictionary
1144 let version = hexpm::version::Range::new(format!("== {}", &config.version))
1145 .expect("== {version} should be a valid range");
1146 let _ = provided.insert(
1147 config.name,
1148 ProvidedPackage {
1149 version: config.version,
1150 source: package_source,
1151 requirements,
1152 },
1153 );
1154 // Return the version
1155 Ok(version)
1156}
1157
1158/// Unlocks specified packages and their unique dependencies.
1159///
1160/// If a manifest is provided, it also unlocks indirect dependencies that are
1161/// not required by any other package or the root project.
1162pub fn unlock_packages(
1163 locked: &mut HashMap<EcoString, Version>,
1164 packages_to_unlock: &[EcoString],
1165 manifest: Option<&Manifest>,
1166) -> Result<()> {
1167 if let Some(manifest) = manifest {
1168 let mut packages_to_unlock: Vec<EcoString> = packages_to_unlock.to_vec();
1169
1170 while let Some(package_name) = packages_to_unlock.pop() {
1171 if locked.remove(&package_name).is_some()
1172 && let Some(package) = manifest.packages.iter().find(|p| p.name == package_name)
1173 {
1174 let deps_to_unlock = find_deps_to_unlock(package, locked, manifest);
1175 packages_to_unlock.extend(deps_to_unlock);
1176 }
1177 }
1178 } else {
1179 for package_name in packages_to_unlock {
1180 let _ = locked.remove(package_name);
1181 }
1182 }
1183
1184 Ok(())
1185}
1186
1187/// Identifies which dependencies of a package should be unlocked.
1188///
1189/// A dependency is eligible for unlocking if it is currently locked,
1190/// is not a root dependency, and is not required by any locked package.
1191fn find_deps_to_unlock(
1192 package: &ManifestPackage,
1193 locked: &HashMap<EcoString, Version>,
1194 manifest: &Manifest,
1195) -> Vec<EcoString> {
1196 package
1197 .requirements
1198 .iter()
1199 .filter(|&dep| {
1200 locked.contains_key(dep)
1201 && !manifest.requirements.contains_key(dep)
1202 && manifest
1203 .packages
1204 .iter()
1205 .all(|p| !locked.contains_key(&p.name) || !p.requirements.contains(dep))
1206 })
1207 .cloned()
1208 .collect()
1209}
1210
1211/// Determine the information to add to the manifest for a specific package
1212async fn lookup_package(
1213 name: String,
1214 version: Version,
1215 provided: &HashMap<EcoString, ProvidedPackage>,
1216) -> Result<ManifestPackage> {
1217 match provided.get(name.as_str()) {
1218 Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
1219 None => {
1220 let config = hexpm::Config::new();
1221 let release =
1222 hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?;
1223 let build_tools = release
1224 .meta
1225 .build_tools
1226 .iter()
1227 .map(|s| EcoString::from(s.as_str()))
1228 .collect_vec();
1229 let requirements = release
1230 .requirements
1231 .keys()
1232 .map(|s| EcoString::from(s.as_str()))
1233 .collect_vec();
1234 Ok(ManifestPackage {
1235 name: name.into(),
1236 version,
1237 otp_app: Some(release.meta.app.into()),
1238 build_tools,
1239 requirements,
1240 source: ManifestPackageSource::Hex {
1241 outer_checksum: Base16Checksum(release.outer_checksum),
1242 },
1243 })
1244 }
1245 }
1246}
1247
1248struct PackageFetcher {
1249 runtime_cache: RefCell<HashMap<String, Rc<hexpm::Package>>>,
1250 runtime: tokio::runtime::Handle,
1251 http: HttpClient,
1252}
1253
1254impl PackageFetcher {
1255 pub fn new(runtime: tokio::runtime::Handle) -> Self {
1256 Self {
1257 runtime_cache: RefCell::new(HashMap::new()),
1258 runtime,
1259 http: HttpClient::new(),
1260 }
1261 }
1262
1263 /// Caches the result of `get_dependencies` so that we don't need to make a network request.
1264 /// Currently dependencies are fetched during initial version resolution, and then during check
1265 /// for major version availability.
1266 fn cache_package(&self, package: &str, result: Rc<hexpm::Package>) {
1267 let mut runtime_cache = self.runtime_cache.borrow_mut();
1268 let _ = runtime_cache.insert(package.to_string(), result);
1269 }
1270}
1271
1272#[derive(Debug)]
1273pub struct Untar;
1274
1275impl Untar {
1276 pub fn boxed() -> Box<Self> {
1277 Box::new(Self)
1278 }
1279}
1280
1281impl TarUnpacker for Untar {
1282 fn io_result_entries<'a>(
1283 &self,
1284 archive: &'a mut tar::Archive<WrappedReader>,
1285 ) -> std::io::Result<tar::Entries<'a, WrappedReader>> {
1286 archive.entries()
1287 }
1288
1289 fn io_result_unpack(
1290 &self,
1291 path: &Utf8Path,
1292 mut archive: tar::Archive<GzDecoder<tar::Entry<'_, WrappedReader>>>,
1293 ) -> std::io::Result<()> {
1294 archive.unpack(path)
1295 }
1296}
1297
1298impl dependency::PackageFetcher for PackageFetcher {
1299 fn get_dependencies(&self, package: &str) -> Result<Rc<hexpm::Package>, PackageFetchError> {
1300 {
1301 let runtime_cache = self.runtime_cache.borrow();
1302 let result = runtime_cache.get(package);
1303
1304 if let Some(result) = result {
1305 return Ok(result.clone());
1306 }
1307 }
1308
1309 tracing::debug!(package = package, "looking_up_hex_package");
1310 let config = hexpm::Config::new();
1311 let request = hexpm::repository_v2_get_package_request(package, None, &config);
1312 let response = self
1313 .runtime
1314 .block_on(self.http.send(request))
1315 .map_err(PackageFetchError::fetch_error)?;
1316
1317 let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
1318 .map_err(|e| PackageFetchError::from_api_error(e, package))?;
1319 let pkg = Rc::new(pkg);
1320 let pkg_ref = Rc::clone(&pkg);
1321 self.cache_package(package, pkg);
1322 Ok(pkg_ref)
1323 }
1324}