nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
44 kB
1322 lines
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2021 The Gleam contributors
3
4mod dependency_manager;
5
6use std::{
7 cell::RefCell,
8 collections::{HashMap, HashSet},
9 io::ErrorKind,
10 process::Command,
11 rc::Rc,
12 time::Instant,
13};
14
15use camino::{Utf8Path, Utf8PathBuf};
16use ecow::{EcoString, eco_format};
17use flate2::read::GzDecoder;
18use gleam_core::{
19 Error, Result,
20 build::{Mode, SourceFingerprint, Target, Telemetry},
21 config::PackageConfig,
22 dependency::{self, PackageFetchError},
23 error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
24 hex::{self, HEXPM_PUBLIC_KEY},
25 io::{HttpClient as _, TarUnpacker, WrappedReader},
26 manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, PackageChanges},
27 paths::ProjectPaths,
28 requirement::Requirement,
29};
30use hexpm::version::Version;
31use itertools::Itertools;
32use same_file::is_same_file;
33use strum::IntoEnumIterator;
34
35pub use dependency_manager::DependencyManagerConfig;
36
37#[cfg(test)]
38mod tests;
39
40use crate::{
41 TreeOptions,
42 build_lock::{BuildLock, Guard},
43 cli,
44 fs::{self, ProjectIO},
45 http::HttpClient,
46 text_layout::space_table,
47};
48
49struct Symbols {
50 down: &'static str,
51 tee: &'static str,
52 ell: &'static str,
53 right: &'static str,
54}
55
56static UTF8_SYMBOLS: Symbols = Symbols {
57 down: "│",
58 tee: "├",
59 ell: "└",
60 right: "─",
61};
62
63/// When set to `Yes`, the cli will check for major version updates of direct dependencies and
64/// print them to the console if the major versions are not upgradeable due to constraints.
65#[derive(Debug, Clone, Copy)]
66pub enum CheckMajorVersions {
67 Yes,
68 No,
69}
70
71pub fn list(paths: &ProjectPaths) -> Result<()> {
72 let (_, manifest) = get_manifest_details(paths)?;
73 list_manifest_packages(std::io::stdout(), manifest)
74}
75
76pub fn tree(paths: &ProjectPaths, options: TreeOptions) -> Result<()> {
77 let (config, manifest) = get_manifest_details(paths)?;
78
79 // Initialize the root package since it is not part of the manifest
80 let root_package = ManifestPackage {
81 build_tools: vec![],
82 name: config.name.clone(),
83 requirements: config.all_direct_dependencies()?.keys().cloned().collect(),
84 version: config.version.clone(),
85 source: ManifestPackageSource::Local {
86 path: paths.root().to_path_buf(),
87 },
88 otp_app: None,
89 };
90
91 // Get the manifest packages and add the root package to the vec
92 let mut packages = manifest.packages.iter().cloned().collect_vec();
93 packages.push(root_package);
94
95 list_package_and_dependencies_tree(std::io::stdout(), options, packages.clone(), config.name)
96}
97
98fn get_manifest_details(paths: &ProjectPaths) -> Result<(PackageConfig, Manifest)> {
99 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
100 let config = crate::config::root_config(paths)?;
101 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
102 let dependency_manager = DependencyManagerConfig {
103 use_manifest: UseManifest::Yes,
104 check_major_versions: CheckMajorVersions::No,
105 }
106 .into_dependency_manager(
107 runtime.handle().clone(),
108 package_fetcher,
109 cli::Reporter::new(),
110 Mode::Dev,
111 );
112 let manifest = dependency_manager
113 .resolve_versions(paths, &config, Vec::new())?
114 .manifest;
115 Ok((config, manifest))
116}
117
118fn list_manifest_packages<W: std::io::Write>(mut buffer: W, manifest: Manifest) -> Result<()> {
119 let packages = manifest
120 .packages
121 .into_iter()
122 .map(|package| vec![package.name.to_string(), package.version.to_string()])
123 .collect_vec();
124 let out = space_table(&["Package", "Version"], packages);
125
126 write!(buffer, "{out}").map_err(|e| Error::StandardIo {
127 action: StandardIoAction::Write,
128 err: Some(e.kind()),
129 })
130}
131
132fn list_package_and_dependencies_tree<W: std::io::Write>(
133 mut buffer: W,
134 options: TreeOptions,
135 packages: Vec<ManifestPackage>,
136 root_package_name: EcoString,
137) -> Result<()> {
138 let mut invert = false;
139
140 let package: Option<&ManifestPackage> = if let Some(input_package_name) = options.package {
141 packages.iter().find(|p| p.name == input_package_name)
142 } else if let Some(input_package_name) = options.invert {
143 invert = true;
144 packages.iter().find(|p| p.name == input_package_name)
145 } else {
146 packages.iter().find(|p| p.name == root_package_name)
147 };
148
149 if let Some(package) = package {
150 let tree = Vec::from([eco_format!("{0} v{1}", package.name, package.version)]);
151 let tree =
152 list_dependencies_tree(tree, package.clone(), packages, EcoString::new(), invert);
153
154 tree.iter()
155 .try_for_each(|line| writeln!(buffer, "{line}"))
156 .map_err(|e| Error::StandardIo {
157 action: StandardIoAction::Write,
158 err: Some(e.kind()),
159 })
160 } else {
161 writeln!(buffer, "Package not found. Please check the package name.").map_err(|e| {
162 Error::StandardIo {
163 action: StandardIoAction::Write,
164 err: Some(e.kind()),
165 }
166 })
167 }
168}
169
170fn list_dependencies_tree(
171 mut tree: Vec<EcoString>,
172 package: ManifestPackage,
173 packages: Vec<ManifestPackage>,
174 accum: EcoString,
175 invert: bool,
176) -> Vec<EcoString> {
177 let dependencies = packages
178 .iter()
179 .filter(|p| {
180 (invert && p.requirements.contains(&package.name))
181 || (!invert && package.requirements.contains(&p.name))
182 })
183 .cloned()
184 .collect_vec();
185
186 let dependencies = dependencies.iter().sorted().enumerate();
187
188 let deps_length = dependencies.len();
189 for (index, dependency) in dependencies {
190 let is_last = index == deps_length - 1;
191 let prefix = if is_last {
192 UTF8_SYMBOLS.ell
193 } else {
194 UTF8_SYMBOLS.tee
195 };
196
197 tree.push(eco_format!(
198 "{0}{1}{2}{2} {3} v{4}",
199 accum.clone(),
200 prefix,
201 UTF8_SYMBOLS.right,
202 dependency.name,
203 dependency.version
204 ));
205
206 let accum = accum.clone() + (if !is_last { UTF8_SYMBOLS.down } else { " " }) + " ";
207
208 tree = list_dependencies_tree(
209 tree.clone(),
210 dependency.clone(),
211 packages.clone(),
212 accum.clone(),
213 invert,
214 );
215 }
216
217 tree
218}
219
220pub fn outdated(paths: &ProjectPaths) -> Result<()> {
221 let (_, manifest) = get_manifest_details(paths)?;
222
223 let total_packages = manifest
224 .packages
225 .iter()
226 .filter(|package| matches!(package.source, ManifestPackageSource::Hex { .. }))
227 .count();
228
229 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
230 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
231
232 let version_updates = dependency::check_for_version_updates(&manifest, &package_fetcher);
233
234 print!(
235 "{}",
236 pretty_print_outdated_versions(total_packages, version_updates)
237 );
238
239 Ok(())
240}
241
242#[derive(Debug, Clone, Copy)]
243pub enum UseManifest {
244 Yes,
245 No,
246}
247
248pub fn update(paths: &ProjectPaths, packages: Vec<String>) -> Result<()> {
249 let use_manifest = if packages.is_empty() {
250 UseManifest::No
251 } else {
252 UseManifest::Yes
253 };
254
255 // Update specific packages
256 _ = resolve_and_download(
257 paths,
258 cli::Reporter::new(),
259 None,
260 packages.into_iter().map(EcoString::from).collect(),
261 DependencyManagerConfig {
262 use_manifest,
263 check_major_versions: CheckMajorVersions::Yes,
264 },
265 )?;
266
267 Ok(())
268}
269
270/// Edit the manifest.toml file in this proejct, removing all extra requirements and packages
271/// that are no longer present in the gleam.toml config.
272pub fn cleanup<Telem: Telemetry>(paths: &ProjectPaths, telemetry: Telem) -> Result<Manifest> {
273 let span = tracing::info_span!("remove_deps");
274 let _enter = span.enter();
275
276 // We do this before acquiring the build lock so that we don't create the
277 // build directory if there is no gleam.toml
278 crate::config::ensure_config_exists(paths)?;
279
280 let lock = BuildLock::new_packages(paths)?;
281 let _guard: Guard = lock.lock(&telemetry)?;
282
283 // Read the project config
284 let config = crate::root_config(paths)?;
285 let old_manifest = read_manifest_from_disc(paths)?;
286 let mut manifest = old_manifest.clone();
287
288 remove_extra_requirements(&config, &mut manifest)?;
289
290 // Remove any packages that are no longer required due to manifest changes
291 let local = LocalPackages::read_from_disc(paths)?;
292 remove_extra_packages(paths, &local, &manifest, &telemetry)?;
293
294 // Record new state of the packages directory
295 tracing::debug!("writing_manifest_toml");
296 write_manifest_to_disc(paths, &manifest)?;
297 LocalPackages::from_manifest(&manifest).write_to_disc(paths)?;
298
299 let changes = PackageChanges::between_manifests(&old_manifest, &manifest);
300 telemetry.resolved_package_versions(&changes);
301
302 Ok(manifest)
303}
304
305/// Remove requirements and unneeded packages from manifest that are no longer present in config.
306fn remove_extra_requirements(config: &PackageConfig, manifest: &mut Manifest) -> Result<()> {
307 // "extra requirements" are all packages that are requirements in the manifest, but no longer
308 // part of the gleam.toml config.
309 let is_extra_requirement = |name: &EcoString| {
310 !config.dev_dependencies.contains_key(name) && !config.dependencies.contains_key(name)
311 };
312
313 // If a requirement is also used as a dependency, we do not want to force-unlock it.
314 // If the dependents get deleted as well, this transitive dependency will be dropped.
315 let is_unlockable_requirement = |name: &EcoString| {
316 manifest
317 .packages
318 .iter()
319 .all(|p| !p.requirements.contains(name))
320 };
321
322 let extra_requirements = manifest
323 .requirements
324 .keys()
325 .filter(|&name| is_extra_requirement(name) && is_unlockable_requirement(name))
326 .cloned()
327 .collect::<Vec<_>>();
328
329 manifest
330 .requirements
331 .retain(|name, _| !is_extra_requirement(name));
332
333 // Unlock all packages that we we want to remove - this removes them and all unneeded
334 // dependencies from `locked`.
335 let mut locked = config.locked(Some(manifest))?;
336 unlock_packages(&mut locked, extra_requirements.as_slice(), Some(manifest))?;
337 // Remove all unlocked packages from the manifest - these are truly no longer needed.
338 manifest
339 .packages
340 .retain(|package| locked.contains_key(&package.name));
341
342 Ok(())
343}
344
345pub fn parse_gleam_add_specifier(package: &str) -> Result<(EcoString, Requirement)> {
346 let Some((package, version)) = package.split_once('@') else {
347 // Default to the latest version available.
348 return Ok((
349 package.into(),
350 Requirement::hex(">= 0.0.0").expect("'>= 0.0.0' should be a valid pubgrub range"),
351 ));
352 };
353
354 // Parse the major and minor from the provided semantic version.
355 let parts = version.split('.').collect::<Vec<_>>();
356 let major = match parts.first() {
357 Some(major) => Ok(major),
358 None => Err(Error::InvalidVersionFormat {
359 input: package.to_string(),
360 error: "Failed to parse semantic major version".to_string(),
361 }),
362 }?;
363 let minor = match parts.get(1) {
364 Some(minor) => minor,
365 None => "0",
366 };
367
368 // Using the major version specifier, calculate the maximum
369 // allowable version (i.e., the next major version).
370 let Ok(num) = major.parse::<usize>() else {
371 return Err(Error::InvalidVersionFormat {
372 input: version.to_string(),
373 error: "Failed to parse semantic major version as integer".to_string(),
374 });
375 };
376
377 let max_ver = [&(num + 1).to_string(), "0", "0"].join(".");
378
379 // Pad the provided version specifier with zeros map to a Hex version.
380 let requirement = match parts.len() {
381 1 | 2 => {
382 let min_ver = [major, minor, "0"].join(".");
383 Requirement::hex(&[">=", &min_ver, "and", "<", &max_ver].join(" "))
384 }
385 3 => Requirement::hex(version),
386 n => {
387 return Err(Error::InvalidVersionFormat {
388 input: version.to_string(),
389 error: format!(
390 "Expected up to 3 numbers in version specifier (MAJOR.MINOR.PATCH), found {n}"
391 ),
392 });
393 }
394 }?;
395
396 Ok((package.into(), requirement))
397}
398
399pub fn resolve_and_download<Telem: Telemetry>(
400 paths: &ProjectPaths,
401 telemetry: Telem,
402 new_package: Option<(Vec<(EcoString, Requirement)>, bool)>,
403 packages_to_update: Vec<EcoString>,
404 config: DependencyManagerConfig,
405) -> Result<Manifest> {
406 // Start event loop so we can run async functions to call the Hex API
407 let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio async runtime");
408 let package_fetcher = PackageFetcher::new(runtime.handle().clone());
409
410 let dependency_manager = config.into_dependency_manager(
411 runtime.handle().clone(),
412 package_fetcher,
413 telemetry,
414 Mode::Dev,
415 );
416
417 dependency_manager.resolve_and_download_versions(paths, new_package, packages_to_update)
418}
419
420fn format_versions_and_extract_longest_parts(
421 versions: dependency::PackageVersionDiffs,
422) -> Vec<Vec<String>> {
423 versions
424 .iter()
425 .map(|(name, (v1, v2))| vec![name.to_string(), v1.to_string(), v2.to_string()])
426 .sorted()
427 .collect_vec()
428}
429
430fn pretty_print_major_versions_available(versions: dependency::PackageVersionDiffs) -> String {
431 let versions = format_versions_and_extract_longest_parts(versions);
432
433 format!(
434 "\nThe following dependencies have new major versions available:\n\n{}",
435 space_table(&["Package", "Current", "Latest"], &versions)
436 )
437}
438
439fn pretty_print_version_updates(versions: dependency::PackageVersionDiffs) -> EcoString {
440 let versions = format_versions_and_extract_longest_parts(versions);
441 space_table(&["Package", "Current", "Latest"], &versions)
442}
443
444fn pretty_print_outdated_versions(
445 total_packages: usize,
446 versions: dependency::PackageVersionDiffs,
447) -> EcoString {
448 let summary = eco_format!(
449 "{} of {} packages have newer versions available.",
450 versions.len(),
451 total_packages
452 );
453
454 if versions.is_empty() {
455 eco_format!("{}\n", summary)
456 } else {
457 eco_format!("{}\n\n{}", summary, pretty_print_version_updates(versions))
458 }
459}
460
461async fn add_missing_packages<Telem: Telemetry>(
462 paths: &ProjectPaths,
463 fs: Box<ProjectIO>,
464 manifest: &Manifest,
465 local: &LocalPackages,
466 project_name: EcoString,
467 telemetry: &Telem,
468) -> Result<(), Error> {
469 let missing_packages = local.missing_local_packages(manifest, &project_name);
470
471 let mut num_to_download = 0;
472
473 let missing_git_packages = missing_packages
474 .iter()
475 .copied()
476 .filter(|package| package.is_git())
477 .inspect(|_| {
478 num_to_download += 1;
479 })
480 .collect_vec();
481
482 let mut missing_hex_packages = missing_packages
483 .iter()
484 .copied()
485 .filter(|package| package.is_hex())
486 .inspect(|_| {
487 num_to_download += 1;
488 })
489 .peekable();
490
491 // If we need to download at-least one package
492 if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() {
493 let http = HttpClient::boxed();
494 let downloader = hex::Downloader::new(fs.clone(), fs, http, Untar::boxed(), paths.clone());
495 let start = Instant::now();
496 telemetry.downloading_package("packages");
497 downloader
498 .download_hex_packages(missing_hex_packages, &project_name)
499 .await?;
500 for package in missing_git_packages {
501 let ManifestPackageSource::Git { repo, commit } = &package.source else {
502 continue;
503 };
504 let _ = download_git_package(&package.name, repo, commit, paths)?;
505 }
506 telemetry.packages_downloaded(start, num_to_download);
507 }
508
509 Ok(())
510}
511
512fn remove_extra_packages<Telem: Telemetry>(
513 paths: &ProjectPaths,
514 local: &LocalPackages,
515 manifest: &Manifest,
516 telemetry: &Telem,
517) -> Result<()> {
518 let _guard = BuildLock::lock_all_build(paths, telemetry)?;
519
520 for (package_name, version) in local.extra_local_packages(manifest) {
521 // TODO: test
522 // Delete the package source
523 let path = paths.build_packages_package(&package_name);
524 if path.exists() {
525 tracing::debug!(package=%package_name, version=%version, "removing_unneeded_package");
526 fs::delete_directory(&path)?;
527 }
528
529 // TODO: test
530 // Delete any build artefacts for the package
531 for mode in Mode::iter() {
532 for target in Target::iter() {
533 let name = manifest
534 .packages
535 .iter()
536 .find(|p| p.name == package_name)
537 .map(|p| p.application_name().as_str())
538 .unwrap_or(package_name.as_str());
539 let path = paths.build_directory_for_package(mode, target, name);
540 if path.exists() {
541 tracing::debug!(package=%package_name, version=%version, "deleting_build_cache");
542 fs::delete_directory(&path)?;
543 }
544 }
545 }
546 }
547 Ok(())
548}
549
550fn read_manifest_from_disc(paths: &ProjectPaths) -> Result<Manifest> {
551 tracing::debug!("reading_manifest_toml");
552 let manifest_path = paths.manifest();
553 let toml = fs::read(&manifest_path)?;
554 let manifest = toml::from_str(&toml).map_err(|e| Error::FileIo {
555 action: FileIoAction::Parse,
556 kind: FileKind::File,
557 path: manifest_path.clone(),
558 err: Some(e.to_string()),
559 })?;
560 Ok(manifest)
561}
562
563fn write_manifest_to_disc(paths: &ProjectPaths, manifest: &Manifest) -> Result<()> {
564 let path = paths.manifest();
565 fs::write(&path, &manifest.to_toml(paths.root()))
566}
567
568// This is the container for locally pinned packages, representing the current contents of
569// the `project/build/packages` directory.
570// For descriptions of packages provided by paths and git deps, see the ProvidedPackage struct.
571// The same package may appear in both at different times.
572#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
573struct LocalPackages {
574 #[serde(deserialize_with = "gleam_core::config::map_with_package_name_keys::deserialize")]
575 packages: HashMap<EcoString, Version>,
576}
577
578impl LocalPackages {
579 pub fn extra_local_packages(&self, manifest: &Manifest) -> Vec<(EcoString, Version)> {
580 let manifest_packages: HashSet<_> = manifest
581 .packages
582 .iter()
583 .map(|p| (&p.name, &p.version))
584 .collect();
585 self.packages
586 .iter()
587 .filter(|(n, v)| !manifest_packages.contains(&(&EcoString::from(*n), v)))
588 .map(|(n, v)| (n.clone(), v.clone()))
589 .collect()
590 }
591
592 pub fn missing_local_packages<'a>(
593 &self,
594 manifest: &'a Manifest,
595 root: &str,
596 ) -> Vec<&'a ManifestPackage> {
597 manifest
598 .packages
599 .iter()
600 // We don't need to download the root package
601 .filter(|p| p.name != root)
602 // We don't need to download local packages because we use the linked source directly
603 .filter(|p| !p.is_local())
604 // We don't need to download packages which we have the correct version of
605 .filter(|p| self.packages.get(p.name.as_str()) != Some(&p.version))
606 .collect()
607 }
608
609 pub fn read_from_disc(paths: &ProjectPaths) -> Result<Self> {
610 let path = paths.build_packages_toml();
611 if !path.exists() {
612 return Ok(Self {
613 packages: HashMap::new(),
614 });
615 }
616 let toml = fs::read(&path)?;
617 toml::from_str(&toml).map_err(|e| Error::FileIo {
618 action: FileIoAction::Parse,
619 kind: FileKind::File,
620 path: path.clone(),
621 err: Some(e.to_string()),
622 })
623 }
624
625 pub fn write_to_disc(&self, paths: &ProjectPaths) -> Result<()> {
626 let path = paths.build_packages_toml();
627 let toml = toml::to_string(&self).expect("packages.toml serialization");
628 fs::write(&path, &toml)
629 }
630
631 pub fn from_manifest(manifest: &Manifest) -> Self {
632 Self {
633 packages: manifest
634 .packages
635 .iter()
636 .map(|p| (p.name.clone(), p.version.clone()))
637 .collect(),
638 }
639 }
640}
641
642#[test]
643fn local_packages_deserialise_ok() {
644 let toml = r#"
645[packages]
646gleam_stdlib = "1.0.0"
647gleam_otp = "1.1.0"
648"#;
649 let packages: LocalPackages = toml::from_str(toml).unwrap();
650 assert_eq!(
651 packages,
652 LocalPackages {
653 packages: HashMap::from_iter([
654 ("gleam_stdlib".into(), Version::new(1, 0, 0)),
655 ("gleam_otp".into(), Version::new(1, 1, 0)),
656 ])
657 }
658 )
659}
660
661#[test]
662fn local_packages_deserialise_invalid_name() {
663 let toml = r#"
664[packages]
665gleam_stdlib = "1.0.0"
666"../../stuff" = "1.1.0"
667"#;
668 let error = toml::from_str::<LocalPackages>(toml)
669 .expect_err("should fail to deserialise because of invalid name");
670 insta::assert_snapshot!(insta::internals::AutoName, error.to_string());
671}
672
673fn is_same_requirements(
674 requirements1: &HashMap<EcoString, Requirement>,
675 requirements2: &HashMap<EcoString, Requirement>,
676 root_path: &Utf8Path,
677) -> Result<bool> {
678 if requirements1.len() != requirements2.len() {
679 return Ok(false);
680 }
681
682 for (key, requirement1) in requirements1 {
683 if !same_requirements(requirement1, requirements2.get(key), root_path)? {
684 return Ok(false);
685 }
686 }
687
688 Ok(true)
689}
690
691/// Returns true if all path dependency configs are unchanged since last build.
692///
693/// If any of the path dependency configs have changed that means that we need to
694/// re-perform dependency resolution, as their dependencies could have changed
695/// themselves.
696///
697/// We use gleam.toml rather than manifest.toml as:
698///
699/// 1. The dependency requirements could have changed but resolution not have been
700/// run in that package yet, so the manifest would be the same, resulting in us
701/// failing to detect that resolution is required.
702///
703/// 2. Dependency manifests are not used in any way, so a change in the manifest
704/// may not have any impact on this package.
705///
706/// This does mean that changes unrelated to the path dependency's dependencies
707/// will trigger resolution, but gleam.toml is edited rarely, and no-change
708/// resolution is fast enough, so that's OK.
709///
710/// Note: This does not check path dependencies of path dependencies! Changes to
711/// their configs will fail to be picked up. To resolve this we would need to keep
712/// a list of all the path dependencies in the project, instead of only the direct
713/// path dependencies.
714///
715fn path_dependency_configs_unchanged(
716 requirements: &HashMap<EcoString, Requirement>,
717 paths: &ProjectPaths,
718) -> Result<bool> {
719 for (name, requirement) in requirements {
720 let Requirement::Path { path } = requirement else {
721 continue;
722 };
723
724 let config_path = paths.path_dependency_gleam_toml_path(path);
725 let fingerprint_path = paths.dependency_gleam_toml_fingerprint_path(name.as_str());
726
727 // Check mtimes before hashing, to avoid extra work
728 if fingerprint_path.exists() {
729 let config_time = fs::modification_time(&config_path)?;
730 let fingerprint_time = fs::modification_time(&fingerprint_path)?;
731 if config_time <= fingerprint_time {
732 continue;
733 }
734 };
735
736 let config_text = fs::read(&config_path)?;
737 let current_fingerprint = SourceFingerprint::new(&config_text).to_numerical_string();
738
739 // If cached hash file doesn't exist, this is the first time we're checking this dependency
740 if !fingerprint_path.exists() {
741 // Save the current hash for future comparisons
742 fs::write(&fingerprint_path, ¤t_fingerprint)?;
743 return Ok(false);
744 }
745
746 let previous_fingerprint = fs::read(&fingerprint_path)?;
747
748 if previous_fingerprint != current_fingerprint {
749 tracing::debug!("path_dependency_config_changed_forcing_rebuild");
750 fs::write(&fingerprint_path, ¤t_fingerprint)?;
751 return Ok(false);
752 }
753 }
754
755 Ok(true)
756}
757
758fn same_requirements(
759 requirement1: &Requirement,
760 requirement2: Option<&Requirement>,
761 root_path: &Utf8Path,
762) -> Result<bool> {
763 let (left, right) = match (requirement1, requirement2) {
764 (Requirement::Path { path: path1 }, Some(Requirement::Path { path: path2 })) => {
765 let left = fs::canonicalise(&root_path.join(path1))?;
766 let right = fs::canonicalise(&root_path.join(path2))?;
767 (left, right)
768 }
769 (_, Some(requirement2)) => return Ok(requirement1 == requirement2),
770 (_, None) => return Ok(false),
771 };
772
773 Ok(left == right)
774}
775
776#[derive(Clone, Eq, PartialEq, Debug)]
777struct ProvidedPackage {
778 version: Version,
779 source: ProvidedPackageSource,
780 requirements: HashMap<EcoString, hexpm::version::Range>,
781}
782
783#[derive(Clone, Eq, Debug)]
784enum ProvidedPackageSource {
785 Git { repo: EcoString, commit: EcoString },
786 Local { path: Utf8PathBuf },
787}
788
789impl ProvidedPackage {
790 fn to_hex_package(&self, name: &EcoString) -> hexpm::Package {
791 let requirements = self
792 .requirements
793 .iter()
794 .map(|(name, version)| {
795 (
796 name.as_str().into(),
797 hexpm::Dependency {
798 requirement: version.clone(),
799 optional: false,
800 app: None,
801 repository: None,
802 },
803 )
804 })
805 .collect();
806 let release = hexpm::Release {
807 version: self.version.clone(),
808 requirements,
809 retirement_status: None,
810 outer_checksum: vec![],
811 meta: (),
812 };
813 hexpm::Package {
814 name: name.as_str().into(),
815 repository: "local".into(),
816 releases: vec![release],
817 }
818 }
819
820 fn to_manifest_package(&self, name: &str) -> ManifestPackage {
821 let mut package = ManifestPackage {
822 name: name.into(),
823 version: self.version.clone(),
824 otp_app: None, // Note, this will probably need to be set to something eventually
825 build_tools: vec!["gleam".into()],
826 requirements: self.requirements.keys().cloned().collect(),
827 source: self.source.to_manifest_package_source(),
828 };
829 package.requirements.sort();
830 package
831 }
832}
833
834impl ProvidedPackageSource {
835 fn to_manifest_package_source(&self) -> ManifestPackageSource {
836 match self {
837 Self::Git { repo, commit } => ManifestPackageSource::Git {
838 repo: repo.clone(),
839 commit: commit.clone(),
840 },
841 Self::Local { path } => ManifestPackageSource::Local { path: path.clone() },
842 }
843 }
844
845 fn to_toml(&self) -> String {
846 match self {
847 Self::Git { repo, commit } => {
848 format!(r#"{{ repo: "{repo}", commit: "{commit}" }}"#)
849 }
850 Self::Local { path } => {
851 format!(r#"{{ path: "{path}" }}"#)
852 }
853 }
854 }
855}
856
857impl PartialEq for ProvidedPackageSource {
858 fn eq(&self, other: &Self) -> bool {
859 match (self, other) {
860 (Self::Local { path: own_path }, Self::Local { path: other_path }) => {
861 is_same_file(own_path, other_path).unwrap_or(false)
862 }
863
864 (
865 Self::Git {
866 repo: own_repo,
867 commit: own_commit,
868 },
869 Self::Git {
870 repo: other_repo,
871 commit: other_commit,
872 },
873 ) => own_repo == other_repo && own_commit == other_commit,
874
875 (Self::Git { .. }, Self::Local { .. }) | (Self::Local { .. }, Self::Git { .. }) => {
876 false
877 }
878 }
879 }
880}
881
882/// Provide a package from a local project
883fn provide_local_package(
884 package_name: EcoString,
885 package_path: &Utf8Path,
886 parent_path: &Utf8Path,
887 project_paths: &ProjectPaths,
888 provided: &mut HashMap<EcoString, ProvidedPackage>,
889 parents: &mut Vec<EcoString>,
890) -> Result<hexpm::version::Range> {
891 let package_path = if package_path.is_absolute() {
892 package_path.to_path_buf()
893 } else {
894 fs::canonicalise(&parent_path.join(package_path))?
895 };
896
897 let package_source = ProvidedPackageSource::Local {
898 path: package_path.clone(),
899 };
900 provide_package(
901 package_name,
902 package_path,
903 package_source,
904 project_paths,
905 provided,
906 parents,
907 )
908}
909
910fn execute_command(command: &mut Command) -> Result<std::process::Output> {
911 let result = command.output();
912 match result {
913 Ok(output) if output.status.success() => Ok(output),
914 Ok(output) => {
915 let reason = match String::from_utf8(output.stderr) {
916 Ok(stderr) => ShellCommandFailureReason::ShellCommandError(stderr),
917 Err(_) => ShellCommandFailureReason::Unknown,
918 };
919 Err(Error::ShellCommand {
920 program: "git".into(),
921 reason,
922 })
923 }
924 Err(error) => Err(match error.kind() {
925 ErrorKind::NotFound => Error::ShellProgramNotFound {
926 program: "git".into(),
927 os: fs::get_os(),
928 },
929
930 other => Error::ShellCommand {
931 program: "git".into(),
932 reason: ShellCommandFailureReason::IoError(other),
933 },
934 }),
935 }
936}
937
938/// Downloads a git package from a remote repository. The commands that are run
939/// looks like this:
940///
941/// ```sh
942/// git init
943/// git remote remove origin
944/// git remote add origin <repo>
945/// git fetch origin
946/// git checkout <ref>
947/// git rev-parse HEAD
948/// ```
949///
950/// This is somewhat inefficient as we have to fetch the entire git history before
951/// switching to the exact commit we want. There a few alternatives to this:
952///
953/// - `git clone --depth 1 --branch="<ref>"` This works, but only allows us to use
954/// branch names as refs, however we want to allow commit hashes as well.
955/// - `git fetch --depth 1 origin <ref>` Similarly, this imposes an unwanted
956/// restriction. `git fetch` only allows branch names or full commit hashes,
957/// but we want to allow partial hashes as well.
958///
959/// Since Git dependencies will be used quite rarely, this option was settled upon
960/// because it allows branch names, full and partial commit hashes as refs.
961///
962/// In the future we can optimise this more, for example first checking if we
963/// are already checked out to the commit stored in the manifest, or by only
964/// fetching the history without the objects to resolve partial commit hashes.
965/// For now though this is good enough until it become an actual performance
966/// problem.
967///
968fn download_git_package(
969 package_name: &str,
970 repo: &str,
971 ref_: &str,
972 project_paths: &ProjectPaths,
973) -> Result<EcoString> {
974 let package_path = project_paths.build_packages_package(package_name);
975
976 // If the package path exists but is not inside a git work tree, we need to
977 // remove the directory because running `git init` in a non-empty directory
978 // followed by `git checkout ...` is an error. See
979 // https://github.com/gleam-lang/gleam/issues/4488 for details.
980 if !fs::is_git_work_tree_root(&package_path) {
981 fs::delete_directory(&package_path)?;
982 }
983
984 fs::mkdir(&package_path)?;
985
986 let _ = execute_command(Command::new("git").arg("init").current_dir(&package_path))?;
987
988 // If this directory already exists, but the remote URL has been edited in
989 // `gleam.toml` without a `gleam clean`, `git remote add` will fail, causing
990 // the remote to be stuck as the original value. Here we remove the remote
991 // first, which ensures that `git remote add` properly add the remote each
992 // time. If this fails, that means we haven't set the remote in the first
993 // place, so we can safely ignore the error.
994 let _ = Command::new("git")
995 .arg("remote")
996 .arg("remove")
997 .arg("origin")
998 .current_dir(&package_path)
999 .output();
1000
1001 let _ = execute_command(
1002 Command::new("git")
1003 .arg("remote")
1004 .arg("add")
1005 .arg("origin")
1006 .arg(repo)
1007 .current_dir(&package_path),
1008 )?;
1009
1010 let _ = execute_command(
1011 Command::new("git")
1012 .arg("fetch")
1013 .arg("origin")
1014 .current_dir(&package_path),
1015 )?;
1016
1017 let _ = execute_command(
1018 Command::new("git")
1019 .arg("checkout")
1020 .arg(ref_)
1021 .current_dir(&package_path),
1022 )?;
1023
1024 let output = execute_command(
1025 Command::new("git")
1026 .arg("rev-parse")
1027 .arg("HEAD")
1028 .current_dir(&package_path),
1029 )?;
1030
1031 let commit = String::from_utf8(output.stdout)
1032 .expect("Output should be UTF-8")
1033 .trim()
1034 .into();
1035
1036 Ok(commit)
1037}
1038
1039/// Provide a package from a git repository
1040fn provide_git_package(
1041 package_name: EcoString,
1042 repo: &str,
1043 // A git ref, such as a branch name, commit hash or tag name
1044 ref_: &str,
1045 project_paths: &ProjectPaths,
1046 provided: &mut HashMap<EcoString, ProvidedPackage>,
1047 parents: &mut Vec<EcoString>,
1048) -> Result<hexpm::version::Range> {
1049 let commit = download_git_package(&package_name, repo, ref_, project_paths)?;
1050
1051 let package_source = ProvidedPackageSource::Git {
1052 repo: repo.into(),
1053 commit,
1054 };
1055
1056 let package_path = fs::canonicalise(&project_paths.build_packages_package(&package_name))?;
1057
1058 provide_package(
1059 package_name,
1060 package_path,
1061 package_source,
1062 project_paths,
1063 provided,
1064 parents,
1065 )
1066}
1067
1068/// Adds a gleam project located at a specific path to the list of "provided packages"
1069fn provide_package(
1070 package_name: EcoString,
1071 package_path: Utf8PathBuf,
1072 package_source: ProvidedPackageSource,
1073 project_paths: &ProjectPaths,
1074 provided: &mut HashMap<EcoString, ProvidedPackage>,
1075 parents: &mut Vec<EcoString>,
1076) -> Result<hexpm::version::Range> {
1077 // Return early if a package cycle is detected
1078 if parents.contains(&package_name) {
1079 let mut last_cycle = parents
1080 .split(|p| p == &package_name)
1081 .next_back()
1082 .unwrap_or_default()
1083 .to_vec();
1084 last_cycle.push(package_name);
1085 return Err(Error::PackageCycle {
1086 packages: last_cycle,
1087 });
1088 }
1089 // Check that we do not have a cached version of this package already
1090 match provided.get(&package_name) {
1091 Some(package) if package.source == package_source => {
1092 // This package has already been provided from this source, return the version
1093 let version = hexpm::version::Range::new(format!("== {}", &package.version))
1094 .expect("== {version} should be a valid range");
1095 return Ok(version);
1096 }
1097 Some(package) => {
1098 // This package has already been provided from a different source which conflicts
1099 return Err(Error::ProvidedDependencyConflict {
1100 package: package_name.into(),
1101 source_1: package_source.to_toml(),
1102 source_2: package.source.to_toml(),
1103 });
1104 }
1105 None => (),
1106 }
1107 // Load the package
1108 let config = crate::config::read(package_path.join("gleam.toml"))?;
1109 // Check that we are loading the correct project
1110 if config.name != package_name {
1111 return Err(Error::WrongDependencyProvided {
1112 expected: package_name.into(),
1113 path: package_path.to_path_buf(),
1114 found: config.name.into(),
1115 });
1116 };
1117 // Walk the requirements of the package
1118 let mut requirements = HashMap::new();
1119 parents.push(package_name);
1120 for (name, requirement) in config.dependencies.into_iter() {
1121 let version = match requirement {
1122 Requirement::Hex { version } => version,
1123 Requirement::Path { path } => {
1124 // Recursively walk local packages
1125 provide_local_package(
1126 name.clone(),
1127 &path,
1128 &package_path,
1129 project_paths,
1130 provided,
1131 parents,
1132 )?
1133 }
1134 Requirement::Git { git, ref_ } => {
1135 provide_git_package(name.clone(), &git, &ref_, project_paths, provided, parents)?
1136 }
1137 };
1138 let _ = requirements.insert(name, version);
1139 }
1140 let _ = parents.pop();
1141 // Add the package to the provided packages dictionary
1142 let version = hexpm::version::Range::new(format!("== {}", &config.version))
1143 .expect("== {version} should be a valid range");
1144 let _ = provided.insert(
1145 config.name,
1146 ProvidedPackage {
1147 version: config.version,
1148 source: package_source,
1149 requirements,
1150 },
1151 );
1152 // Return the version
1153 Ok(version)
1154}
1155
1156/// Unlocks specified packages and their unique dependencies.
1157///
1158/// If a manifest is provided, it also unlocks indirect dependencies that are
1159/// not required by any other package or the root project.
1160pub fn unlock_packages(
1161 locked: &mut HashMap<EcoString, Version>,
1162 packages_to_unlock: &[EcoString],
1163 manifest: Option<&Manifest>,
1164) -> Result<()> {
1165 if let Some(manifest) = manifest {
1166 let mut packages_to_unlock: Vec<EcoString> = packages_to_unlock.to_vec();
1167
1168 while let Some(package_name) = packages_to_unlock.pop() {
1169 if locked.remove(&package_name).is_some()
1170 && let Some(package) = manifest.packages.iter().find(|p| p.name == package_name)
1171 {
1172 let deps_to_unlock = find_deps_to_unlock(package, locked, manifest);
1173 packages_to_unlock.extend(deps_to_unlock);
1174 }
1175 }
1176 } else {
1177 for package_name in packages_to_unlock {
1178 let _ = locked.remove(package_name);
1179 }
1180 }
1181
1182 Ok(())
1183}
1184
1185/// Identifies which dependencies of a package should be unlocked.
1186///
1187/// A dependency is eligible for unlocking if it is currently locked,
1188/// is not a root dependency, and is not required by any locked package.
1189fn find_deps_to_unlock(
1190 package: &ManifestPackage,
1191 locked: &HashMap<EcoString, Version>,
1192 manifest: &Manifest,
1193) -> Vec<EcoString> {
1194 package
1195 .requirements
1196 .iter()
1197 .filter(|&dep| {
1198 locked.contains_key(dep)
1199 && !manifest.requirements.contains_key(dep)
1200 && manifest
1201 .packages
1202 .iter()
1203 .all(|p| !locked.contains_key(&p.name) || !p.requirements.contains(dep))
1204 })
1205 .cloned()
1206 .collect()
1207}
1208
1209/// Determine the information to add to the manifest for a specific package
1210async fn lookup_package(
1211 name: String,
1212 version: Version,
1213 provided: &HashMap<EcoString, ProvidedPackage>,
1214) -> Result<ManifestPackage> {
1215 match provided.get(name.as_str()) {
1216 Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
1217 None => {
1218 let config = hexpm::Config::new();
1219 let release =
1220 hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?;
1221 let build_tools = release
1222 .meta
1223 .build_tools
1224 .iter()
1225 .map(|s| EcoString::from(s.as_str()))
1226 .collect_vec();
1227 let requirements = release
1228 .requirements
1229 .keys()
1230 .map(|s| EcoString::from(s.as_str()))
1231 .collect_vec();
1232 Ok(ManifestPackage {
1233 name: name.into(),
1234 version,
1235 otp_app: Some(release.meta.app.into()),
1236 build_tools,
1237 requirements,
1238 source: ManifestPackageSource::Hex {
1239 outer_checksum: Base16Checksum(release.outer_checksum),
1240 },
1241 })
1242 }
1243 }
1244}
1245
1246struct PackageFetcher {
1247 runtime_cache: RefCell<HashMap<String, Rc<hexpm::Package>>>,
1248 runtime: tokio::runtime::Handle,
1249 http: HttpClient,
1250}
1251
1252impl PackageFetcher {
1253 pub fn new(runtime: tokio::runtime::Handle) -> Self {
1254 Self {
1255 runtime_cache: RefCell::new(HashMap::new()),
1256 runtime,
1257 http: HttpClient::new(),
1258 }
1259 }
1260
1261 /// Caches the result of `get_dependencies` so that we don't need to make a network request.
1262 /// Currently dependencies are fetched during initial version resolution, and then during check
1263 /// for major version availability.
1264 fn cache_package(&self, package: &str, result: Rc<hexpm::Package>) {
1265 let mut runtime_cache = self.runtime_cache.borrow_mut();
1266 let _ = runtime_cache.insert(package.to_string(), result);
1267 }
1268}
1269
1270#[derive(Debug)]
1271pub struct Untar;
1272
1273impl Untar {
1274 pub fn boxed() -> Box<Self> {
1275 Box::new(Self)
1276 }
1277}
1278
1279impl TarUnpacker for Untar {
1280 fn io_result_entries<'a>(
1281 &self,
1282 archive: &'a mut tar::Archive<WrappedReader>,
1283 ) -> std::io::Result<tar::Entries<'a, WrappedReader>> {
1284 archive.entries()
1285 }
1286
1287 fn io_result_unpack(
1288 &self,
1289 path: &Utf8Path,
1290 mut archive: tar::Archive<GzDecoder<tar::Entry<'_, WrappedReader>>>,
1291 ) -> std::io::Result<()> {
1292 archive.unpack(path)
1293 }
1294}
1295
1296impl dependency::PackageFetcher for PackageFetcher {
1297 fn get_dependencies(&self, package: &str) -> Result<Rc<hexpm::Package>, PackageFetchError> {
1298 {
1299 let runtime_cache = self.runtime_cache.borrow();
1300 let result = runtime_cache.get(package);
1301
1302 if let Some(result) = result {
1303 return Ok(result.clone());
1304 }
1305 }
1306
1307 tracing::debug!(package = package, "looking_up_hex_package");
1308 let config = hexpm::Config::new();
1309 let request = hexpm::repository_v2_get_package_request(package, None, &config);
1310 let response = self
1311 .runtime
1312 .block_on(self.http.send(request))
1313 .map_err(PackageFetchError::fetch_error)?;
1314
1315 let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
1316 .map_err(|e| PackageFetchError::from_api_error(e, package))?;
1317 let pkg = Rc::new(pkg);
1318 let pkg_ref = Rc::clone(&pkg);
1319 self.cache_package(package, pkg);
1320 Ok(pkg_ref)
1321 }
1322}