Login
[x]
Log in using an account from:
Fedora Account System
Red Hat Associate
Red Hat Customer
Or login using a Red Hat Bugzilla account
Forgot Password
Login:
Hide Forgot
Create an Account
Red Hat Bugzilla – Attachment 711657 Details for
Bug 892063
deja-dup writes gigabytes of data to /tmp, which doesn't play well with F18 tmpfs default: No space left in '/'.
[?]
New
Simple Search
Advanced Search
My Links
Browse
Requests
Reports
Current State
Search
Tabular reports
Graphical reports
Duplicates
Other Reports
User Changes
Plotly Reports
Bug Status
Bug Severity
Non-Defaults
|
Product Dashboard
Help
Page Help!
Bug Writing Guidelines
What's new
Browser Support Policy
5.0.4.rh83 Release notes
FAQ
Guides index
User guide
Web Services
Contact
Legal
This site requires JavaScript to be enabled to function correctly, please enable it.
[patch]
DD 24.0 tempdir patch
dd24-tempdir.patch (text/plain), 18.77 KB, created by
Michael Terry
on 2013-03-18 00:32:52 UTC
(
hide
)
Description:
DD 24.0 tempdir patch
Filename:
MIME Type:
Creator:
Michael Terry
Created:
2013-03-18 00:32:52 UTC
Size:
18.77 KB
patch
obsolete
>=== modified file 'common/CommonUtils.vala' >--- common/CommonUtils.vala 2012-08-20 01:16:15 +0000 >+++ common/CommonUtils.vala 2013-03-17 23:50:06 +0000 >@@ -444,6 +444,9 @@ > var unused_backend = DejaDup.Backend.get_default(); > unused_backend = null; > >+ // And cleanup from any previous runs >+ clean_tempdirs.begin(); >+ > return true; > } > >@@ -582,5 +585,127 @@ > return date; > } > >+public bool ensure_directory_exists(string path) >+{ >+ var gfile = File.new_for_path(path); >+ try { >+ if (gfile.make_directory_with_parents()) >+ return true; >+ } >+ catch (IOError.EXISTS e) { >+ return true; // ignore >+ } >+ catch (Error e) { >+ warning("%s\n", e.message); >+ } >+ return false; >+} >+ >+// By default, duplicity uses normal tmp folders like /tmp to store its >+// in-process files. These can get quite large, especially when restoring. >+// You may need up to twice the size of the largest source file. >+// Because /tmp may not be super large, especially on systems that use >+// tmpfs by default (e.g. Fedora 18), we try to use a tempdir that is on >+// the same partition as the source files. >+public async string get_tempdir() >+{ >+ var tempdirs = get_tempdirs(); >+ >+ // First, decide the "main include". Assume that if $HOME >+ // is present, that is it. Else, the first include we find decides it. >+ // This is admittedly fast and loose, but our primary concern is just >+ // avoiding silly choices like tmpfs or tiny special /tmp partitions. >+ var settings = get_settings(); >+ var include_val = settings.get_value(INCLUDE_LIST_KEY); >+ var include_list = parse_dir_list(include_val.get_strv()); >+ File main_include = null; >+ var home = File.new_for_path(Environment.get_home_dir()); >+ foreach (var include in include_list) { >+ if (include.equal(home)) { >+ main_include = include; >+ break; >+ } >+ else if (main_include == null) >+ main_include = include; >+ } >+ if (main_include == null) >+ return tempdirs[0]; >+ >+ // Grab that include's fs ID >+ string filesystem_id; >+ try { >+ var info = yield main_include.query_info_async(FileAttribute.ID_FILESYSTEM, >+ FileQueryInfoFlags.NONE); >+ filesystem_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM); >+ } >+ catch (Error e) { >+ return tempdirs[0]; >+ } >+ >+ // Then, see which of our possible tempdirs matches that partition. >+ foreach (var tempdir in tempdirs) { >+ string temp_id; >+ ensure_directory_exists(tempdir); >+ try { >+ var gfile = File.new_for_path(tempdir); >+ var info = yield gfile.query_info_async(FileAttribute.ID_FILESYSTEM, >+ FileQueryInfoFlags.NONE); >+ temp_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM); >+ } >+ catch (Error e) { >+ continue; >+ } >+ if (temp_id == filesystem_id) >+ return tempdir; >+ } >+ >+ // Fallback to simply using the highest preferred tempdir >+ return tempdirs[0]; >+} >+ >+public string[] get_tempdirs() >+{ >+ var tempdir = Environment.get_variable("DEJA_DUP_TEMPDIR"); >+ if (tempdir != null && tempdir != "") >+ return {tempdir}; >+ // Prefer directories that have their own cleanup logic in case ours isn't >+ // run for a while. (e.g. /tmp every boot, /var/tmp every now and then) >+ return {Environment.get_tmp_dir(), "/var/tmp", >+ Path.build_filename(Environment.get_user_cache_dir(), Config.PACKAGE, >+ "tmp")}; >+} >+ >+public async void clean_tempdirs() >+{ >+ var tempdirs = get_tempdirs(); >+ const int NUM_ENUMERATED = 16; >+ foreach (var tempdir in tempdirs) { >+ var gfile = File.new_for_path(tempdir); >+ >+ // Now try to find and delete all files that start with "duplicity-" >+ try { >+ var enumerator = yield gfile.enumerate_children_async( >+ FileAttribute.STANDARD_NAME, >+ FileQueryInfoFlags.NOFOLLOW_SYMLINKS, >+ Priority.DEFAULT, null); >+ while (true) { >+ var infos = yield enumerator.next_files_async(NUM_ENUMERATED, >+ Priority.DEFAULT, null); >+ foreach (FileInfo info in infos) { >+ if (info.get_name().has_prefix("duplicity-")) { >+ var child = gfile.get_child(info.get_name()); >+ yield new DejaDup.RecursiveDelete(child).start_async(); >+ } >+ } >+ if (infos.length() != NUM_ENUMERATED) >+ break; >+ } >+ } >+ catch (Error e) { >+ // No worries >+ } >+ } >+} >+ > } // end namespace > > >=== modified file 'common/Operation.vala' >--- common/Operation.vala 2012-08-20 01:16:15 +0000 >+++ common/Operation.vala 2013-03-17 23:50:06 +0000 >@@ -223,6 +223,7 @@ > finished = true; > > unclaim_bus(); >+ yield DejaDup.clean_tempdirs(); > > done(success, cancelled, detail); > } > >=== modified file 'common/OperationBackup.vala' >--- common/OperationBackup.vala 2012-08-13 19:59:00 +0000 >+++ common/OperationBackup.vala 2013-03-17 23:50:06 +0000 >@@ -116,11 +116,11 @@ > rv.append(Path.build_filename(dir, ".xsession-errors")); > } > >- // Some problematic directories like /tmp and /proc should be left alone >- dir = Environment.get_tmp_dir(); >- if (dir != null) >- rv.append(dir); >- >+ // Skip all of our temporary directories >+ foreach (var tempdir in DejaDup.get_tempdirs()) >+ rv.append(tempdir); >+ >+ // Skip kernel directories > rv.append("/proc"); > rv.append("/sys"); > > >=== modified file 'tests/runner/runner.vala' >--- tests/runner/runner.vala 2012-08-21 01:29:56 +0000 >+++ tests/runner/runner.vala 2013-03-18 00:15:03 +0000 >@@ -51,7 +51,14 @@ > Environment.set_variable("DEJA_DUP_TOOLS_PATH", "../../tools/duplicity", true); > Environment.set_variable("DEJA_DUP_TEST_MOCKSCRIPT", Path.build_filename(dir, "mockscript"), true); > Environment.set_variable("XDG_CACHE_HOME", cachedir, true); >- Environment.set_variable("PATH", "./mock:" + Environment.get_variable("PATH"), true); >+ Environment.set_variable("PATH", >+ "./mock:" + >+ Environment.get_variable("DEJA_DUP_TEST_PATH"), >+ true); >+ >+ var tempdir = Path.build_filename(dir, "tmp"); >+ DejaDup.ensure_directory_exists(tempdir); >+ Environment.set_variable("DEJA_DUP_TEMPDIR", tempdir, true); > > var settings = DejaDup.get_settings(); > settings.set_string(DejaDup.BACKEND_KEY, "file"); >@@ -61,11 +68,9 @@ > > void backup_teardown() > { >- var path = Environment.get_variable("PATH"); >- if (path.has_prefix("./mock:")) { >- path = path.substring(7); >- Environment.set_variable("PATH", path, true); >- } >+ Environment.set_variable("PATH", >+ Environment.get_variable("DEJA_DUP_TEST_PATH"), >+ true); > > var file = File.new_for_path(Environment.get_variable("DEJA_DUP_TEST_MOCKSCRIPT")); > if (file.query_exists(null)) { >@@ -112,14 +117,22 @@ > > var archive = tmp_archive ? "?" : "%s/deja-dup".printf(cachedir); > >+ string enc_str = ""; >+ if (!encrypted) >+ enc_str = "--no-encryption "; >+ >+ var tempdir = Path.build_filename(test_home, "tmp"); >+ >+ var end_str = "%s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--tempdir=%s' '--log-fd=?'".printf(enc_str, archive, tempdir); >+ > if (mode == Mode.CLEANUP) >- return "cleanup '--force' 'file://%s' '--gio' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive); >+ return "cleanup '--force' 'file://%s' '--gio' %s".printf(backupdir, end_str); > else if (mode == Mode.RESTORE) >- return "'restore' '--gio' '--force' 'file://%s' '%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, restoredir, encrypted ? "" : "'--no-encryption' ", archive); >+ return "'restore' '--gio' '--force' 'file://%s' '%s' %s".printf(backupdir, restoredir, end_str); > else if (mode == Mode.VERIFY) >- return "'restore' '--file-to-restore=%s/deja-dup/metadata' '--gio' '--force' 'file://%s' '%s/deja-dup/metadata' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(cachedir.substring(1), backupdir, cachedir, encrypted ? "" : "'--no-encryption' ", archive); >+ return "'restore' '--file-to-restore=%s/deja-dup/metadata' '--gio' '--force' 'file://%s' '%s/deja-dup/metadata' %s".printf(cachedir.substring(1), backupdir, cachedir, end_str); > else if (mode == Mode.LIST) >- return "'list-current-files' '--gio' 'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive); >+ return "'list-current-files' '--gio' 'file://%s' %s".printf(backupdir, end_str); > > string source_str = ""; > if (mode == Mode.DRY || mode == Mode.BACKUP) >@@ -129,10 +142,6 @@ > if (mode == Mode.DRY) > dry_str = "--dry-run "; > >- string enc_str = ""; >- if (!encrypted) >- enc_str = "--no-encryption "; >- > string args = ""; > > if (br.is_full && !br.is_first && (mode == Mode.BACKUP || mode == Mode.DRY)) >@@ -147,7 +156,7 @@ > string[] excludes1 = {"~/Downloads", "~/.local/share/Trash", "~/.xsession-errors", "~/.thumbnails", "~/.Private", "~/.gvfs", "~/.adobe/Flash_Player/AssetCache"}; > > var user = Environment.get_user_name(); >- string[] excludes2 = {"/home/.ecryptfs/%s/.Private".printf(user), "/sys", "/proc", "/tmp"}; >+ string[] excludes2 = {"/home/.ecryptfs/%s/.Private".printf(user), "/sys", "/proc", tempdir}; > > foreach (string ex in excludes1) { > ex = ex.replace("~", Environment.get_home_dir()); >@@ -166,7 +175,7 @@ > args += "'--exclude=%s/deja-dup' '--exclude=%s' '--exclude=**' ".printf(cachedir, cachedir); > } > >- args += "%s%s'--gio' %s'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--log-fd=?'".printf(extra, dry_str, source_str, backupdir, enc_str, archive); >+ args += "%s%s'--gio' %s'file://%s' %s".printf(extra, dry_str, source_str, backupdir, end_str); > > return args; > } >@@ -175,6 +184,8 @@ > { > public delegate void OpCallback (DejaDup.Operation op); > public DejaDup.Operation op = null; >+ public string path = null; >+ public string script = null; > public string? init_error = null; > public bool success = true; > public bool cancelled = false; >@@ -188,6 +199,12 @@ > > public void run() > { >+ if (script != null) >+ run_script(script); >+ >+ if (path != null) >+ Environment.set_variable("PATH", path, true); >+ > string header, msg; > if (!DejaDup.initialize(out header, out msg)) { > if (header + "\n" + msg != init_error) >@@ -298,9 +315,13 @@ > string replace_keywords(string in) > { > var home = Environment.get_home_dir(); >+ var mockdir = "./mock"; > var cachedir = Environment.get_variable("XDG_CACHE_HOME"); > var test_home = Environment.get_variable("DEJA_DUP_TEST_HOME"); >+ var path = Environment.get_variable("PATH"); > return in.replace("@HOME@", home). >+ replace("@MOCK_DIR@", mockdir). >+ replace("@PATH@", path). > replace("@XDG_CACHE_HOME@", cachedir). > replace("@TEST_HOME@", test_home); > } >@@ -351,6 +372,10 @@ > br.error_detail = keyfile.get_string(group, "ErrorDetail"); > if (keyfile.has_key(group, "Passphrases")) > br.passphrases = keyfile.get_integer(group, "Passphrases"); >+ if (keyfile.has_key(group, "Path")) >+ br.path = replace_keywords(keyfile.get_string(group, "Path")); >+ if (keyfile.has_key(group, "Script")) >+ br.script = replace_keywords(keyfile.get_string(group, "Script")); > if (keyfile.has_key(group, "Settings")) { > var settings_list = keyfile.get_string_list(group, "Settings"); > var settings = DejaDup.get_settings(); >@@ -449,10 +474,15 @@ > }; > } > >- if (script != null) >- dupscript += "\n" + "SCRIPT: " + script; >- else if (mode == Mode.VERIFY) >- dupscript += "\n" + "SCRIPT: mkdir -p %s/deja-dup/metadata; echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir); >+ var verify_script = "mkdir -p %s/deja-dup/metadata && echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir); >+ if (mode == Mode.VERIFY) >+ dupscript += "\n" + "SCRIPT: " + verify_script; >+ if (script != null) { >+ if (mode == Mode.VERIFY) >+ dupscript += " && " + script; >+ else >+ dupscript += "\n" + "SCRIPT: " + script; >+ } > > if (passphrase) > dupscript += "\n" + "PASSPHRASE: test"; >@@ -523,6 +553,10 @@ > script = args[1]; > Environment.set_variable("DEJA_DUP_TEST_SCRIPT", script, true); > >+ // Save PATH, as tests might reset it on us >+ Environment.set_variable("DEJA_DUP_TEST_PATH", >+ Environment.get_variable("PATH"), true); >+ > var parts = script.split("/"); > var suitename = parts[parts.length - 2]; > var testname = parts[parts.length - 1].split(".")[0]; > >=== added file 'tests/scripts/clean-tempdir.test' >--- tests/scripts/clean-tempdir.test 1970-01-01 00:00:00 +0000 >+++ tests/scripts/clean-tempdir.test 2013-03-17 23:50:25 +0000 >@@ -0,0 +1,12 @@ >+# Make sure we clean up the tempdir that we give to duplicity after we're done >+ >+[Operation] >+Type=backup >+Script=mkdir -p @TEST_HOME@/tmp/duplicity-blarg @TEST_HOME@/tmp/nope >+ >+[Duplicity] >+Runs=status;dry;backup;status-restore;list;verify; >+ >+# Check at end of everything that we cleared the temp files >+[Duplicity verify] >+Script=test ! -e @TEST_HOME@/tmp/duplicity-blarg -a -d @TEST_HOME@/tmp/nope > >=== added file 'tests/scripts/instance-error.test' >--- tests/scripts/instance-error.test 1970-01-01 00:00:00 +0000 >+++ tests/scripts/instance-error.test 2013-03-18 00:21:57 +0000 >@@ -0,0 +1,21 @@ >+# Make sure we correctly bubble up an error message from >+# DuplicityInstance.start(). In this case, an error about spawning duplicity. >+ >+[Operation] >+Type=backup >+# Delete mockscript ourselves, because the runner will notice that it is still >+# there and fail. Since we never run our mock duplicity, it is never cleaned >+# naturally. Also symlink python since the mock duplicity needs it >+Script=mkdir -p @TEST_HOME@/mockcopy && cp -r @MOCK_DIR@/* @TEST_HOME@/mockcopy && ln -s `which python` @TEST_HOME@/mockcopy/python >+Path=@TEST_HOME@/mockcopy >+Success=false >+Error=Failed to execute child process "duplicity" (No such file or directory) >+ >+[Duplicity] >+Runs=status;dry; >+ >+[Duplicity status] >+# We let duplicity go for one operation to let it get initialized (i.e. we >+# don't want to mess up any of its early calls like --version). >+# Also clean the mockscript out since our mock won't be doing it. >+Script=/bin/rm @TEST_HOME@/mockcopy/duplicity @TEST_HOME@/mockscript > >=== modified file 'tests/scripts/verify.test' >--- tests/scripts/verify.test 2012-08-10 18:33:29 +0000 >+++ tests/scripts/verify.test 2013-03-17 23:50:25 +0000 >@@ -7,4 +7,4 @@ > Runs=status;dry;backup;status-restore;list;verify; > > [Duplicity verify] >-Script=mkdir -p @XDG_CACHE_HOME@/deja-dup/metadata; echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README >+Script=echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README > >=== modified file 'tools/duplicity/DuplicityInstance.vala' >--- tools/duplicity/DuplicityInstance.vala 2012-08-20 01:16:15 +0000 >+++ tools/duplicity/DuplicityInstance.vala 2013-03-17 23:50:25 +0000 >@@ -29,8 +29,30 @@ > public bool verbose {get; private set; default = false;} > public string forced_cache_dir {get; set; default = null;} > >- public virtual void start(List<string> argv_in, List<string>? envp_in, >- bool as_root = false) throws Error >+ public async void start(List<string> argv_in, List<string>? envp_in, >+ bool as_root = false) >+ { >+ try { >+ /* Make deep copies of the lists, so if our caller doesn't yield, the >+ lists won't be invalidated. */ >+ var argv = new List<string>(); >+ foreach (var arg in argv_in) >+ argv.append(arg); >+ var envp = new List<string>(); >+ foreach (var env in envp_in) >+ envp.append(env); >+ if (!yield start_internal(argv, envp, as_root)) >+ done(false, false); >+ } >+ catch (Error e) { >+ // Fake a generic message from duplicity >+ message({"ERROR", "1"}, null, e.message); >+ done(false, false); >+ } >+ } >+ >+ async bool start_internal(List<string> argv_in, List<string>? envp_in, >+ bool as_root) throws Error > { > var verbose_str = Environment.get_variable("DEJA_DUP_DEBUG"); > if (verbose_str != null && int.parse(verbose_str) > 0) >@@ -76,46 +98,29 @@ > if (cache_dir == null) > cache_dir = Environment.get_user_cache_dir(); > if (cache_dir != null) { >- bool add_dir = false; >- var cache_file = File.new_for_path(cache_dir); >- cache_file = cache_file.get_child(Config.PACKAGE); >- try { >- if (cache_file.make_directory_with_parents(null)) >- add_dir = true; >- } >- catch (IOError.EXISTS e) { >- add_dir = true; // ignore >- } >- catch (Error e) { >- warning("%s\n", e.message); >- } >- if (add_dir) >- argv.append("--archive-dir=" + cache_file.get_path()); >+ cache_dir = Path.build_filename(cache_dir, Config.PACKAGE); >+ if (DejaDup.ensure_directory_exists(cache_dir)) >+ argv.append("--archive-dir=" + cache_dir); > } >- >+ >+ // Specify tempdir >+ var tempdir = yield DejaDup.get_tempdir(); >+ if (DejaDup.ensure_directory_exists(tempdir)) >+ argv.append("--tempdir=%s".printf(tempdir)); >+ > // Add logging argument > if (as_root) { > // Make log file > int logfd = 0; >- try { >- string logname; >- logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname); >- logfile = File.new_for_path(logname); >- } >- catch (Error e) { >- warning("%s\n", e.message); >- done(false, false); >- return; >- } >- >+ string logname; >+ logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname); >+ logfile = File.new_for_path(logname); > argv.append("--log-file=%s".printf(logfile.get_path())); > } > else { > // Open pipes to communicate with subprocess >- if (Posix.pipe(pipes) != 0) { >- done(false, false); >- return; >- } >+ if (Posix.pipe(pipes) != 0) >+ return false; > > argv.append("--log-fd=%d".printf(pipes[1])); > } >@@ -195,9 +200,10 @@ > if (pipes[1] != -1) > Posix.close(pipes[1]); > >- read_log.begin(); >+ yield read_log(); >+ return true; > } >- >+ > public bool is_started() > { > return (int)child_pid > 0; > >=== modified file 'tools/duplicity/DuplicityJob.vala' >--- tools/duplicity/DuplicityJob.vala 2012-08-21 01:27:58 +0000 >+++ tools/duplicity/DuplicityJob.vala 2013-03-17 23:50:25 +0000 >@@ -1459,13 +1459,7 @@ > } > > /* Start duplicity instance */ >- try { >- inst.start(argv, envp, needs_root); >- } >- catch (Error e) { >- show_error(e.message); >- done(false, false, null); >- } >+ inst.start.begin(argv, envp, needs_root); > } > } > >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
Attachments on
bug 892063
: 711657 |
750751