restore: preserve hardlinks on restore (#492)

This is the simplest (and probably least efficient) way to address
#16

This issue **has been open since 2022**, and it seems potentially like a
way to have very confusing BUGS if you backup this folder:
```sh
wstein@lite:/tmp$ mkdir link
wstein@lite:/tmp$ cd link/
wstein@lite:/tmp/link$ ls
wstein@lite:/tmp/link$ echo "foo" > A.txt
wstein@lite:/tmp/link$ ln A.txt B.txt
wstein@lite:/tmp/link$ echo "bar" >> B.txt
wstein@lite:/tmp/link$ more A.txt
foo
bar
wstein@lite:/tmp/link$ more B.txt
foo
bar
```
then restore it, and suddenly it behave very differently after the
restore.

I'm building software where rustic backup/restore has to actually work
well -- it's not just for emergency recovery, but part of the normal
lifecycle of user data. I ran some big tests of backup/restore, and only
after fixing this hard link issue was the filesystem restored properly.

Fortunately, restore already records inode, device id, and link count
metadata for files. But the restore path recreated every file as an
independent plain file, so **hardlinks were silently de-linked on
restore**.

This PR adds a post-restore hardlink pass keyed by the stored
`(device_id, inode)` identity. After file contents and metadata are
restored, sibling paths in each hardlink group are replaced with
hardlinks to a canonical restored path.
Also add a test using the existing backup fixture that contains a
hardlink pair.

Obviously, it could make good sense to close this if:
- you don't like that it isn't the globally optimal way to solve this
problem (since it restores each linked file, then combines them, rather
than restoring only one). It was just the minimal way to do this that at
the end preserves the links.
- I didn't worry at all about non-POSIX

If you close this, it would probably be good though to mention clearly
on the main rustic README page that hardlinks silently break on backup
--> restore, as it can lead to major bugs for users, and it's only
something they might find via careful testing / debugging of errors
popping up later (that was the case for me). "hard link" isn't mentioned
anywhere on main rustic README or issue tracker (it's in this
rustic_core issue).

For anybody who does need this, I put a release here for linux:

https://github.com/sagemathinc/rustic/releases

Thanks!

Author: williamstein

crates/core/src/backend/local_destination.rs

@@ -102,6 +102,12 @@ pub enum LocalDestinationErrorKind {
         filename: PathBuf,
         source: std::io::Error,
     },
+    /// failed to create hardlink from `{source_path:?}` to `{filename:?}` with `{source:?}`
+    HardLinkingFailed {
+        source_path: PathBuf,
+        filename: PathBuf,
+        source: std::io::Error,
+    },
 }
 
 pub(crate) type LocalDestinationResult<T> = Result<T, LocalDestinationErrorKind>;
@@ -791,4 +797,37 @@ impl LocalDestination {
             .map_err(LocalDestinationErrorKind::CouldNotWriteToBuffer)?;
         Ok(())
     }
+
+    /// Create a hardlink `item` pointing to `source_item`, both relative to the base path.
+    ///
+    /// # Arguments
+    ///
+    /// * `source_item` - The already-restored file to link to
+    /// * `item` - The path to create as a hardlink
+    ///
+    /// # Errors
+    ///
+    /// * If the new hardlink does not have a parent directory.
+    /// * If the directory could not be created.
+    /// * If the hardlink could not be created.
+    pub(crate) fn hard_link(
+        &self,
+        source_item: impl AsRef<Path>,
+        item: impl AsRef<Path>,
+    ) -> LocalDestinationResult<()> {
+        let source_path = self.path(source_item);
+        let filename = self.path(item);
+        let dir = filename
+            .parent()
+            .ok_or_else(|| LocalDestinationErrorKind::FileDoesNotHaveParent(filename.clone()))?;
+        fs::create_dir_all(dir).map_err(LocalDestinationErrorKind::DirectoryCreationFailed)?;
+        fs::hard_link(&source_path, &filename).map_err(|err| {
+            LocalDestinationErrorKind::HardLinkingFailed {
+                source_path,
+                filename,
+                source: err,
+            }
+        })?;
+        Ok(())
+    }
 }

crates/core/src/commands/restore.rs

@@ -90,6 +90,12 @@ pub struct RestoreStats {
     pub dirs: FileDirStats,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+struct HardlinkKey {
+    device_id: u64,
+    inode: u64,
+}
+
 /// Restore the repository to the given destination.
 ///
 /// # Type Parameters
@@ -350,7 +356,11 @@ fn restore_metadata(
     dest: &LocalDestination,
 ) -> RusticResult<()> {
     let mut dir_stack = Vec::new();
+    let mut hardlinks = BTreeMap::<HardlinkKey, Vec<PathBuf>>::new();
     while let Some((path, node)) = node_streamer.next().transpose()? {
+        if let Some(key) = hardlink_key(&node) {
+            hardlinks.entry(key).or_default().push(path.clone());
+        }
         match node.node_type {
             NodeType::Dir => {
                 // set metadata for all non-parent paths in stack
@@ -373,6 +383,57 @@ fn restore_metadata(
         set_metadata(dest, opts, &path, &node);
     }
 
+    restore_hardlinks(dest, &hardlinks)?;
+
+    Ok(())
+}
+
+fn hardlink_key(node: &Node) -> Option<HardlinkKey> {
+    matches!(node.node_type, NodeType::File)
+        .then_some(HardlinkKey {
+            device_id: node.meta.device_id,
+            inode: node.meta.inode,
+        })
+        .filter(|key| node.meta.links > 1 && key.device_id != 0 && key.inode != 0)
+}
+
+fn restore_hardlinks(
+    dest: &LocalDestination,
+    hardlinks: &BTreeMap<HardlinkKey, Vec<PathBuf>>,
+) -> RusticResult<()> {
+    for paths in hardlinks.values() {
+        if paths.len() < 2 {
+            continue;
+        }
+
+        let canonical = &paths[0];
+        for path in paths.iter().skip(1) {
+            debug!(
+                "restoring hardlink {} -> {}",
+                path.display(),
+                canonical.display()
+            );
+            let full_path = dest.path(path);
+            dest.remove_file(&full_path).map_err(|err| {
+                RusticError::with_source(
+                    ErrorKind::InputOutput,
+                    "Failed to remove the file `{path}` before recreating its hardlink.",
+                    err,
+                )
+                .attach_context("path", path.display().to_string())
+            })?;
+            dest.hard_link(canonical, path).map_err(|err| {
+                RusticError::with_source(
+                    ErrorKind::InputOutput,
+                    "Failed to recreate the hardlink `{path}` from `{canonical}`.",
+                    err,
+                )
+                .attach_context("path", path.display().to_string())
+                .attach_context("canonical", canonical.display().to_string())
+            })?;
+        }
+    }
+
     Ok(())
 }
 

crates/core/tests/integration/restore.rs

@@ -1 +1,60 @@
+use std::{fs, path::PathBuf, str::FromStr};
 
+#[cfg(not(windows))]
+use std::os::unix::fs::MetadataExt;
+
+use anyhow::Result;
+use pretty_assertions::assert_eq;
+use rstest::rstest;
+use tempfile::tempdir;
+
+use rustic_core::{
+    BackupOptions, LocalDestination, LsOptions, RestoreOptions, repofile::SnapshotFile,
+};
+
+use super::{RepoOpen, TestSource, set_up_repo, tar_gz_testdata};
+
+#[rstest]
+#[cfg(not(windows))]
+fn test_restore_preserves_hardlinks(
+    tar_gz_testdata: Result<TestSource>,
+    set_up_repo: Result<RepoOpen>,
+) -> Result<()> {
+    let (source, repo) = (tar_gz_testdata?, set_up_repo?.to_indexed_ids()?);
+
+    let opts = BackupOptions::default().as_path(PathBuf::from_str("test")?);
+    let _snapshot = repo.backup(&opts, &source.path_list(), SnapshotFile::default())?;
+
+    let repo = repo.to_indexed()?;
+    let node = repo.node_from_snapshot_path("latest", |_| true)?;
+    let ls_opts = LsOptions::default();
+    let ls = repo.ls(&node, &ls_opts)?;
+
+    let restore_dir = tempdir()?;
+    let dest = LocalDestination::new(
+        restore_dir
+            .path()
+            .to_str()
+            .expect("restore path is valid utf-8"),
+        true,
+        !node.is_dir(),
+    )?;
+    let restore_opts = RestoreOptions::default();
+    let plan = repo.prepare_restore(&restore_opts, ls.clone(), &dest, false)?;
+    repo.restore(plan, &restore_opts, ls, &dest)?;
+
+    let hardlink = restore_dir.path().join("test/0/tests/testfile-hardlink");
+    let linked = restore_dir.path().join("test/0/tests/testfile");
+    let symlink = restore_dir.path().join("test/0/tests/testfile-symlink");
+
+    let hardlink_meta = fs::metadata(&hardlink)?;
+    let linked_meta = fs::metadata(&linked)?;
+    assert_eq!(hardlink_meta.dev(), linked_meta.dev());
+    assert_eq!(hardlink_meta.ino(), linked_meta.ino());
+    assert_eq!(hardlink_meta.nlink(), 2);
+    assert_eq!(linked_meta.nlink(), 2);
+    assert_eq!(fs::read_to_string(&hardlink)?, fs::read_to_string(&linked)?);
+    assert_eq!(fs::read_link(&symlink)?, PathBuf::from("testfile"));
+
+    Ok(())
+}