Attachment 711657 Details for Bug 892063 – DD 24.0 tempdir patch

[patch] DD 24.0 tempdir patch

dd24-tempdir.patch (text/plain), 18.77 KB, created by Michael Terry on 2013-03-18 00:32:52 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Michael Terry

Created: 2013-03-18 00:32:52 UTC

Size: 18.77 KB

patch

obsolete

>=== modified file 'common/CommonUtils.vala'
>--- common/CommonUtils.vala	2012-08-20 01:16:15 +0000
>+++ common/CommonUtils.vala	2013-03-17 23:50:06 +0000
>@@ -444,6 +444,9 @@
>   var unused_backend = DejaDup.Backend.get_default();
>   unused_backend = null;
> 
>+  // And cleanup from any previous runs
>+  clean_tempdirs.begin();
>+
>   return true;
> }
> 
>@@ -582,5 +585,127 @@
>   return date;
> }
> 
>+public bool ensure_directory_exists(string path)
>+{
>+  var gfile = File.new_for_path(path);
>+  try {
>+    if (gfile.make_directory_with_parents())
>+      return true;
>+  }
>+  catch (IOError.EXISTS e) {
>+    return true; // ignore
>+  }
>+  catch (Error e) {
>+    warning("%s\n", e.message);
>+  }
>+  return false;
>+}
>+
>+// By default, duplicity uses normal tmp folders like /tmp to store its
>+// in-process files.  These can get quite large, especially when restoring.
>+// You may need up to twice the size of the largest source file.
>+// Because /tmp may not be super large, especially on systems that use
>+// tmpfs by default (e.g. Fedora 18), we try to use a tempdir that is on
>+// the same partition as the source files.
>+public async string get_tempdir()
>+{
>+  var tempdirs = get_tempdirs();
>+
>+  // First, decide the "main include".  Assume that if $HOME
>+  // is present, that is it.  Else, the first include we find decides it.
>+  // This is admittedly fast and loose, but our primary concern is just
>+  // avoiding silly choices like tmpfs or tiny special /tmp partitions.
>+  var settings = get_settings();
>+  var include_val = settings.get_value(INCLUDE_LIST_KEY);
>+  var include_list = parse_dir_list(include_val.get_strv());
>+  File main_include = null;
>+  var home = File.new_for_path(Environment.get_home_dir());
>+  foreach (var include in include_list) {
>+    if (include.equal(home)) {
>+      main_include = include;
>+      break;
>+    }
>+    else if (main_include == null)
>+      main_include = include;
>+  }
>+  if (main_include == null)
>+    return tempdirs[0];
>+
>+  // Grab that include's fs ID
>+  string filesystem_id;
>+  try {
>+    var info = yield main_include.query_info_async(FileAttribute.ID_FILESYSTEM,
>+                                                   FileQueryInfoFlags.NONE);
>+    filesystem_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM);
>+  }
>+  catch (Error e) {
>+    return tempdirs[0];
>+  }
>+
>+  // Then, see which of our possible tempdirs matches that partition.
>+  foreach (var tempdir in tempdirs) {
>+    string temp_id;
>+    ensure_directory_exists(tempdir);
>+    try {
>+      var gfile = File.new_for_path(tempdir);
>+      var info = yield gfile.query_info_async(FileAttribute.ID_FILESYSTEM,
>+                                              FileQueryInfoFlags.NONE);
>+      temp_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM);
>+    }
>+    catch (Error e) {
>+      continue;
>+    }
>+    if (temp_id == filesystem_id)
>+      return tempdir;
>+  }
>+
>+  // Fallback to simply using the highest preferred tempdir
>+  return tempdirs[0];
>+}
>+
>+public string[] get_tempdirs()
>+{
>+  var tempdir = Environment.get_variable("DEJA_DUP_TEMPDIR");
>+  if (tempdir != null && tempdir != "")
>+    return {tempdir};
>+  // Prefer directories that have their own cleanup logic in case ours isn't
>+  // run for a while.  (e.g. /tmp every boot, /var/tmp every now and then)
>+  return {Environment.get_tmp_dir(), "/var/tmp",
>+          Path.build_filename(Environment.get_user_cache_dir(), Config.PACKAGE,
>+                              "tmp")};
>+}
>+
>+public async void clean_tempdirs()
>+{
>+  var tempdirs = get_tempdirs();
>+  const int NUM_ENUMERATED = 16;
>+  foreach (var tempdir in tempdirs) {
>+    var gfile = File.new_for_path(tempdir);
>+
>+    // Now try to find and delete all files that start with "duplicity-"
>+    try {
>+      var enumerator = yield gfile.enumerate_children_async(
>+                         FileAttribute.STANDARD_NAME,
>+                         FileQueryInfoFlags.NOFOLLOW_SYMLINKS,
>+                         Priority.DEFAULT, null);
>+      while (true) {
>+        var infos = yield enumerator.next_files_async(NUM_ENUMERATED,
>+                                                      Priority.DEFAULT, null);
>+        foreach (FileInfo info in infos) {
>+          if (info.get_name().has_prefix("duplicity-")) {
>+            var child = gfile.get_child(info.get_name());
>+            yield new DejaDup.RecursiveDelete(child).start_async();
>+          }
>+        }
>+        if (infos.length() != NUM_ENUMERATED)
>+          break;
>+      }
>+    }
>+    catch (Error e) {
>+      // No worries
>+    }
>+  }
>+}
>+
> } // end namespace
> 
>
>=== modified file 'common/Operation.vala'
>--- common/Operation.vala	2012-08-20 01:16:15 +0000
>+++ common/Operation.vala	2013-03-17 23:50:06 +0000
>@@ -223,6 +223,7 @@
>     finished = true;
> 
>     unclaim_bus();
>+    yield DejaDup.clean_tempdirs();
> 
>     done(success, cancelled, detail);
>   }
>
>=== modified file 'common/OperationBackup.vala'
>--- common/OperationBackup.vala	2012-08-13 19:59:00 +0000
>+++ common/OperationBackup.vala	2013-03-17 23:50:06 +0000
>@@ -116,11 +116,11 @@
>       rv.append(Path.build_filename(dir, ".xsession-errors"));
>     }
>     
>-    // Some problematic directories like /tmp and /proc should be left alone
>-    dir = Environment.get_tmp_dir();
>-    if (dir != null)
>-      rv.append(dir);
>-    
>+    // Skip all of our temporary directories
>+    foreach (var tempdir in DejaDup.get_tempdirs())
>+      rv.append(tempdir);
>+
>+    // Skip kernel directories
>     rv.append("/proc");
>     rv.append("/sys");
>     
>
>=== modified file 'tests/runner/runner.vala'
>--- tests/runner/runner.vala	2012-08-21 01:29:56 +0000
>+++ tests/runner/runner.vala	2013-03-18 00:15:03 +0000
>@@ -51,7 +51,14 @@
>   Environment.set_variable("DEJA_DUP_TOOLS_PATH", "../../tools/duplicity", true);
>   Environment.set_variable("DEJA_DUP_TEST_MOCKSCRIPT", Path.build_filename(dir, "mockscript"), true);
>   Environment.set_variable("XDG_CACHE_HOME", cachedir, true);
>-  Environment.set_variable("PATH", "./mock:" + Environment.get_variable("PATH"), true);
>+  Environment.set_variable("PATH",
>+                           "./mock:" +
>+                             Environment.get_variable("DEJA_DUP_TEST_PATH"),
>+                           true);
>+
>+  var tempdir = Path.build_filename(dir, "tmp");
>+  DejaDup.ensure_directory_exists(tempdir);
>+  Environment.set_variable("DEJA_DUP_TEMPDIR", tempdir, true);
> 
>   var settings = DejaDup.get_settings();
>   settings.set_string(DejaDup.BACKEND_KEY, "file");
>@@ -61,11 +68,9 @@
> 
> void backup_teardown()
> {
>-  var path = Environment.get_variable("PATH");
>-  if (path.has_prefix("./mock:")) {
>-    path = path.substring(7);
>-    Environment.set_variable("PATH", path, true);
>-  }
>+  Environment.set_variable("PATH",
>+                           Environment.get_variable("DEJA_DUP_TEST_PATH"),
>+                           true);
> 
>   var file = File.new_for_path(Environment.get_variable("DEJA_DUP_TEST_MOCKSCRIPT"));
>   if (file.query_exists(null)) {
>@@ -112,14 +117,22 @@
> 
>   var archive = tmp_archive ? "?" : "%s/deja-dup".printf(cachedir);
> 
>+  string enc_str = "";
>+  if (!encrypted)
>+    enc_str = "--no-encryption ";
>+
>+  var tempdir = Path.build_filename(test_home, "tmp");
>+
>+  var end_str = "%s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--tempdir=%s' '--log-fd=?'".printf(enc_str, archive, tempdir);
>+
>   if (mode == Mode.CLEANUP)
>-    return "cleanup '--force' 'file://%s' '--gio' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive);
>+    return "cleanup '--force' 'file://%s' '--gio' %s".printf(backupdir, end_str);
>   else if (mode == Mode.RESTORE)
>-    return "'restore' '--gio' '--force' 'file://%s' '%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, restoredir, encrypted ? "" : "'--no-encryption' ", archive);
>+    return "'restore' '--gio' '--force' 'file://%s' '%s' %s".printf(backupdir, restoredir, end_str);
>   else if (mode == Mode.VERIFY)
>-    return "'restore' '--file-to-restore=%s/deja-dup/metadata' '--gio' '--force' 'file://%s' '%s/deja-dup/metadata' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(cachedir.substring(1), backupdir, cachedir, encrypted ? "" : "'--no-encryption' ", archive);
>+    return "'restore' '--file-to-restore=%s/deja-dup/metadata' '--gio' '--force' 'file://%s' '%s/deja-dup/metadata' %s".printf(cachedir.substring(1), backupdir, cachedir, end_str);
>   else if (mode == Mode.LIST)
>-    return "'list-current-files' '--gio' 'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive);
>+    return "'list-current-files' '--gio' 'file://%s' %s".printf(backupdir, end_str);
> 
>   string source_str = "";
>   if (mode == Mode.DRY || mode == Mode.BACKUP)
>@@ -129,10 +142,6 @@
>   if (mode == Mode.DRY)
>     dry_str = "--dry-run ";
> 
>-  string enc_str = "";
>-  if (!encrypted)
>-    enc_str = "--no-encryption ";
>-
>   string args = "";
> 
>   if (br.is_full && !br.is_first && (mode == Mode.BACKUP || mode == Mode.DRY))
>@@ -147,7 +156,7 @@
>     string[] excludes1 = {"~/Downloads", "~/.local/share/Trash", "~/.xsession-errors", "~/.thumbnails", "~/.Private", "~/.gvfs", "~/.adobe/Flash_Player/AssetCache"};
> 
>     var user = Environment.get_user_name();
>-    string[] excludes2 = {"/home/.ecryptfs/%s/.Private".printf(user), "/sys", "/proc", "/tmp"};
>+    string[] excludes2 = {"/home/.ecryptfs/%s/.Private".printf(user), "/sys", "/proc", tempdir};
> 
>     foreach (string ex in excludes1) {
>       ex = ex.replace("~", Environment.get_home_dir());
>@@ -166,7 +175,7 @@
>     args += "'--exclude=%s/deja-dup' '--exclude=%s' '--exclude=**' ".printf(cachedir, cachedir);
>   }
> 
>-  args += "%s%s'--gio' %s'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(extra, dry_str, source_str, backupdir, enc_str, archive);
>+  args += "%s%s'--gio' %s'file://%s' %s".printf(extra, dry_str, source_str, backupdir, end_str);
> 
>   return args;
> }
>@@ -175,6 +184,8 @@
> {
>   public delegate void OpCallback (DejaDup.Operation op);
>   public DejaDup.Operation op = null;
>+  public string path = null;
>+  public string script = null;
>   public string? init_error = null;
>   public bool success = true;
>   public bool cancelled = false;
>@@ -188,6 +199,12 @@
> 
>   public void run()
>   {
>+    if (script != null)
>+      run_script(script);
>+
>+    if (path != null)
>+      Environment.set_variable("PATH", path, true);
>+
>     string header, msg;
>     if (!DejaDup.initialize(out header, out msg)) {
>       if (header + "\n" + msg != init_error)
>@@ -298,9 +315,13 @@
> string replace_keywords(string in)
> {
>   var home = Environment.get_home_dir();
>+  var mockdir = "./mock";
>   var cachedir = Environment.get_variable("XDG_CACHE_HOME");
>   var test_home = Environment.get_variable("DEJA_DUP_TEST_HOME");
>+  var path = Environment.get_variable("PATH");
>   return in.replace("@HOME@", home).
>+         replace("@MOCK_DIR@", mockdir).
>+         replace("@PATH@", path).
>          replace("@XDG_CACHE_HOME@", cachedir).
>          replace("@TEST_HOME@", test_home);
> }
>@@ -351,6 +372,10 @@
>     br.error_detail = keyfile.get_string(group, "ErrorDetail");
>   if (keyfile.has_key(group, "Passphrases"))
>     br.passphrases = keyfile.get_integer(group, "Passphrases");
>+  if (keyfile.has_key(group, "Path"))
>+    br.path = replace_keywords(keyfile.get_string(group, "Path"));
>+   if (keyfile.has_key(group, "Script"))
>+    br.script = replace_keywords(keyfile.get_string(group, "Script"));
>   if (keyfile.has_key(group, "Settings")) {
>     var settings_list = keyfile.get_string_list(group, "Settings");
>     var settings = DejaDup.get_settings();
>@@ -449,10 +474,15 @@
>     };
>   }
> 
>-  if (script != null)
>-    dupscript += "\n" + "SCRIPT: " + script;
>-  else if (mode == Mode.VERIFY)
>-    dupscript += "\n" + "SCRIPT: mkdir -p %s/deja-dup/metadata; echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir);
>+  var verify_script = "mkdir -p %s/deja-dup/metadata && echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir);
>+  if (mode == Mode.VERIFY)
>+    dupscript += "\n" + "SCRIPT: " + verify_script;
>+  if (script != null) {
>+    if (mode == Mode.VERIFY)
>+      dupscript += " && " + script;
>+    else
>+      dupscript += "\n" + "SCRIPT: " + script;
>+  }
> 
>   if (passphrase)
>     dupscript += "\n" + "PASSPHRASE: test";
>@@ -523,6 +553,10 @@
>     script = args[1];
>   Environment.set_variable("DEJA_DUP_TEST_SCRIPT", script, true);
> 
>+  // Save PATH, as tests might reset it on us
>+  Environment.set_variable("DEJA_DUP_TEST_PATH",
>+                           Environment.get_variable("PATH"), true);
>+
>   var parts = script.split("/");
>   var suitename = parts[parts.length - 2];
>   var testname = parts[parts.length - 1].split(".")[0];
>
>=== added file 'tests/scripts/clean-tempdir.test'
>--- tests/scripts/clean-tempdir.test	1970-01-01 00:00:00 +0000
>+++ tests/scripts/clean-tempdir.test	2013-03-17 23:50:25 +0000
>@@ -0,0 +1,12 @@
>+# Make sure we clean up the tempdir that we give to duplicity after we're done
>+
>+[Operation]
>+Type=backup
>+Script=mkdir -p @TEST_HOME@/tmp/duplicity-blarg @TEST_HOME@/tmp/nope
>+
>+[Duplicity]
>+Runs=status;dry;backup;status-restore;list;verify;
>+
>+# Check at end of everything that we cleared the temp files
>+[Duplicity verify]
>+Script=test ! -e @TEST_HOME@/tmp/duplicity-blarg -a -d @TEST_HOME@/tmp/nope
>
>=== added file 'tests/scripts/instance-error.test'
>--- tests/scripts/instance-error.test	1970-01-01 00:00:00 +0000
>+++ tests/scripts/instance-error.test	2013-03-18 00:21:57 +0000
>@@ -0,0 +1,21 @@
>+# Make sure we correctly bubble up an error message from
>+# DuplicityInstance.start().  In this case, an error about spawning duplicity.
>+
>+[Operation]
>+Type=backup
>+# Delete mockscript ourselves, because the runner will notice that it is still
>+# there and fail.  Since we never run our mock duplicity, it is never cleaned
>+# naturally.  Also symlink python since the mock duplicity needs it
>+Script=mkdir -p @TEST_HOME@/mockcopy && cp -r @MOCK_DIR@/* @TEST_HOME@/mockcopy && ln -s `which python` @TEST_HOME@/mockcopy/python
>+Path=@TEST_HOME@/mockcopy
>+Success=false
>+Error=Failed to execute child process "duplicity" (No such file or directory)
>+
>+[Duplicity]
>+Runs=status;dry;
>+
>+[Duplicity status]
>+# We let duplicity go for one operation to let it get initialized (i.e. we
>+# don't want to mess up any of its early calls like --version).
>+# Also clean the mockscript out since our mock won't be doing it.
>+Script=/bin/rm @TEST_HOME@/mockcopy/duplicity @TEST_HOME@/mockscript
>
>=== modified file 'tests/scripts/verify.test'
>--- tests/scripts/verify.test	2012-08-10 18:33:29 +0000
>+++ tests/scripts/verify.test	2013-03-17 23:50:25 +0000
>@@ -7,4 +7,4 @@
> Runs=status;dry;backup;status-restore;list;verify;
> 
> [Duplicity verify]
>-Script=mkdir -p @XDG_CACHE_HOME@/deja-dup/metadata; echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README
>+Script=echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README
>
>=== modified file 'tools/duplicity/DuplicityInstance.vala'
>--- tools/duplicity/DuplicityInstance.vala	2012-08-20 01:16:15 +0000
>+++ tools/duplicity/DuplicityInstance.vala	2013-03-17 23:50:25 +0000
>@@ -29,8 +29,30 @@
>   public bool verbose {get; private set; default = false;}
>   public string forced_cache_dir {get; set; default = null;}
>   
>-  public virtual void start(List<string> argv_in, List<string>? envp_in,
>-                            bool as_root = false) throws Error
>+  public async void start(List<string> argv_in, List<string>? envp_in,
>+                          bool as_root = false)
>+  {
>+    try {
>+      /* Make deep copies of the lists, so if our caller doesn't yield, the
>+         lists won't be invalidated. */
>+      var argv = new List<string>();
>+      foreach (var arg in argv_in)
>+        argv.append(arg);
>+      var envp = new List<string>();
>+      foreach (var env in envp_in)
>+        envp.append(env);
>+      if (!yield start_internal(argv, envp, as_root))
>+        done(false, false);
>+    }
>+    catch (Error e) {
>+      // Fake a generic message from duplicity
>+      message({"ERROR", "1"}, null, e.message);
>+      done(false, false);
>+    }
>+  }
>+
>+  async bool start_internal(List<string> argv_in, List<string>? envp_in,
>+                            bool as_root) throws Error
>   {
>     var verbose_str = Environment.get_variable("DEJA_DUP_DEBUG");
>     if (verbose_str != null && int.parse(verbose_str) > 0)
>@@ -76,46 +98,29 @@
>     if (cache_dir == null)
>       cache_dir = Environment.get_user_cache_dir();
>     if (cache_dir != null) {
>-      bool add_dir = false;
>-      var cache_file = File.new_for_path(cache_dir);
>-      cache_file = cache_file.get_child(Config.PACKAGE);
>-      try {
>-        if (cache_file.make_directory_with_parents(null))
>-          add_dir = true;
>-      }
>-      catch (IOError.EXISTS e) {
>-        add_dir = true; // ignore
>-      }
>-      catch (Error e) {
>-        warning("%s\n", e.message);
>-      }
>-      if (add_dir)
>-        argv.append("--archive-dir=" + cache_file.get_path());
>+      cache_dir = Path.build_filename(cache_dir, Config.PACKAGE);
>+      if (DejaDup.ensure_directory_exists(cache_dir))
>+        argv.append("--archive-dir=" + cache_dir);
>     }
>-    
>+
>+    // Specify tempdir
>+    var tempdir = yield DejaDup.get_tempdir();
>+    if (DejaDup.ensure_directory_exists(tempdir))
>+      argv.append("--tempdir=%s".printf(tempdir));
>+
>     // Add logging argument
>     if (as_root) {
>       // Make log file
>       int logfd = 0;
>-      try {
>-        string logname;
>-        logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname);
>-        logfile = File.new_for_path(logname);
>-      }
>-      catch (Error e) {
>-        warning("%s\n", e.message);
>-        done(false, false);
>-        return;
>-      }
>-      
>+      string logname;
>+      logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname);
>+      logfile = File.new_for_path(logname);
>       argv.append("--log-file=%s".printf(logfile.get_path()));
>     }
>     else {
>       // Open pipes to communicate with subprocess
>-      if (Posix.pipe(pipes) != 0) {
>-        done(false, false);
>-        return;
>-      }
>+      if (Posix.pipe(pipes) != 0)
>+        return false;
> 
>       argv.append("--log-fd=%d".printf(pipes[1]));
>     }
>@@ -195,9 +200,10 @@
>     if (pipes[1] != -1)
>       Posix.close(pipes[1]);
>     
>-    read_log.begin();
>+    yield read_log();
>+    return true;
>   }
>-  
>+
>   public bool is_started()
>   {
>     return (int)child_pid > 0;
>
>=== modified file 'tools/duplicity/DuplicityJob.vala'
>--- tools/duplicity/DuplicityJob.vala	2012-08-21 01:27:58 +0000
>+++ tools/duplicity/DuplicityJob.vala	2013-03-17 23:50:25 +0000
>@@ -1459,13 +1459,7 @@
>     }
> 
>     /* Start duplicity instance */
>-    try {
>-      inst.start(argv, envp, needs_root);
>-    }
>-    catch (Error e) {
>-      show_error(e.message);
>-      done(false, false, null);
>-    }
>+    inst.start.begin(argv, envp, needs_root);
>   }
> }
> 
>

Actions: View | Diff

Attachments on bug 892063: 711657 | 750751