Closes #1036, Migrate storage directory folders to secondary keys
Also moves orphaned directories into orphaned-files folder in data dir Safety first: Keeps track of moved files, moving them back if there's an error before the end of the upgrade process (since the keys are generated randomly and would be different if recreated), and creates a zotero.moved-files.36.bak file with a list of id/key pairs
This commit is contained in:
Dan Stillman
parent
08279f3ff2
commit
012229552e
4 changed files with 159 additions and 94 deletions
|
|
@ -36,7 +36,9 @@ Zotero.Attachments = new function(){
|
|||
this.createMissingAttachment = createMissingAttachment;
|
||||
this.getFileBaseNameFromItem = getFileBaseNameFromItem;
|
||||
this.createDirectoryForItem = createDirectoryForItem;
|
||||
this.createDirectoryForMissingItem = createDirectoryForMissingItem;
|
||||
this.getStorageDirectory = getStorageDirectory;
|
||||
this.getMissingStorageDirectory = getMissingStorageDirectory;
|
||||
this.getPath = getPath;
|
||||
|
||||
var self = this;
|
||||
|
|
@ -56,6 +58,7 @@ Zotero.Attachments = new function(){
|
|||
attachmentItem.setSource(sourceItemID);
|
||||
attachmentItem.attachmentLinkMode = this.LINK_MODE_IMPORTED_FILE;
|
||||
var itemID = attachmentItem.save();
|
||||
attachmentItem = Zotero.Items.get(itemID);
|
||||
|
||||
// Create directory for attachment files within storage directory
|
||||
var destDir = this.createDirectoryForItem(itemID);
|
||||
|
|
@ -85,10 +88,9 @@ Zotero.Attachments = new function(){
|
|||
|
||||
try {
|
||||
// Clean up
|
||||
if (itemID){
|
||||
var itemDir = Zotero.getStorageDirectory();
|
||||
itemDir.append(itemID);
|
||||
if (itemDir.exists()){
|
||||
if (itemID) {
|
||||
var itemDir = this.getStorageDirectory(itemID);
|
||||
if (itemDir.exists()) {
|
||||
itemDir.remove(true);
|
||||
}
|
||||
}
|
||||
|
|
@ -138,15 +140,17 @@ Zotero.Attachments = new function(){
|
|||
// create a proper item, but at the moment this is only called by
|
||||
// translate.js, which sets the metadata fields itself
|
||||
var itemID = attachmentItem.save();
|
||||
attachmentItem = Zotero.Items.get(itemID)
|
||||
var attachmentKey = attachmentItem.key;
|
||||
|
||||
var storageDir = Zotero.getStorageDirectory();
|
||||
file.parent.copyTo(storageDir, itemID);
|
||||
file.parent.copyTo(storageDir, attachmentKey);
|
||||
|
||||
// Point to copied file
|
||||
var newFile = Components.classes["@mozilla.org/file/local;1"]
|
||||
.createInstance(Components.interfaces.nsILocalFile);
|
||||
newFile.initWithFile(storageDir);
|
||||
newFile.append(itemID);
|
||||
newFile.append(attachmentKey);
|
||||
newFile.append(file.leafName);
|
||||
|
||||
attachmentItem.path = this.getPath(newFile, this.LINK_MODE_IMPORTED_URL);
|
||||
|
|
@ -162,10 +166,9 @@ Zotero.Attachments = new function(){
|
|||
|
||||
try {
|
||||
// Clean up
|
||||
if (itemID){
|
||||
var itemDir = Zotero.getStorageDirectory();
|
||||
itemDir.append(itemID);
|
||||
if (itemDir.exists()){
|
||||
if (itemID) {
|
||||
var itemDir = this.getStorageDirectory(itemID);
|
||||
if (itemDir.exists()) {
|
||||
itemDir.remove(true);
|
||||
}
|
||||
}
|
||||
|
|
@ -260,6 +263,7 @@ Zotero.Attachments = new function(){
|
|||
attachmentItem.attachmentLinkMode = Zotero.Attachments.LINK_MODE_IMPORTED_URL;
|
||||
attachmentItem.attachmentMIMEType = mimeType;
|
||||
var itemID = attachmentItem.save();
|
||||
attachmentItem = Zotero.Items.get(itemID);
|
||||
|
||||
// Add to collections
|
||||
if (parentCollectionIDs){
|
||||
|
|
@ -280,8 +284,6 @@ Zotero.Attachments = new function(){
|
|||
|
||||
wbp.progressListener = new Zotero.WebProgressFinishListener(function(){
|
||||
try {
|
||||
var attachmentItem = Zotero.Items.get(itemID);
|
||||
|
||||
var str = Zotero.File.getSample(file);
|
||||
if (mimeType == 'application/pdf' &&
|
||||
Zotero.MIME.sniffForMIMEType(str) != 'application/pdf') {
|
||||
|
|
@ -291,9 +293,10 @@ Zotero.Attachments = new function(){
|
|||
return;
|
||||
}
|
||||
|
||||
attachmentItem.attachmentPath = Zotero.Attachments.getPath(
|
||||
file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID
|
||||
);
|
||||
attachmentItem.attachmentPath =
|
||||
Zotero.Attachments.getPath(
|
||||
file, Zotero.Attachments.LINK_MODE_IMPORTED_URL
|
||||
);
|
||||
attachmentItem.save();
|
||||
|
||||
Zotero.Notifier.trigger('add', 'item', itemID);
|
||||
|
|
@ -337,10 +340,9 @@ Zotero.Attachments = new function(){
|
|||
try {
|
||||
// Clean up
|
||||
if (itemID) {
|
||||
var destDir = Zotero.getStorageDirectory();
|
||||
destDir.append(itemID);
|
||||
if (destDir.exists()) {
|
||||
destDir.remove(true);
|
||||
var itemDir = this.getStorageDirectory(itemID);
|
||||
if (itemDir.exists()) {
|
||||
itemDir.remove(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -538,8 +540,9 @@ Zotero.Attachments = new function(){
|
|||
wpdDOMSaver.init(file.path, document);
|
||||
wpdDOMSaver.saveHTMLDocument();
|
||||
|
||||
var path = this.getPath(file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID);
|
||||
attachmentItem.attachmentPath = path;
|
||||
attachmentItem.attachmentPath = this.getPath(
|
||||
file, Zotero.Attachments.LINK_MODE_IMPORTED_URL
|
||||
);
|
||||
attachmentItem.save();
|
||||
}
|
||||
else {
|
||||
|
|
@ -555,8 +558,10 @@ Zotero.Attachments = new function(){
|
|||
var nsIURL = ioService.newURI(url, null, null);
|
||||
wbp.progressListener = new Zotero.WebProgressFinishListener(function () {
|
||||
try {
|
||||
var path = this.getPath(file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID);
|
||||
attachmentItem.attachmentPath = path;
|
||||
attachmentItem.attachmentPath = this.getPath(
|
||||
file,
|
||||
Zotero.Attachments.LINK_MODE_IMPORTED_URL
|
||||
);
|
||||
attachmentItem.save();
|
||||
|
||||
Zotero.Notifier.trigger('add', 'item', itemID);
|
||||
|
|
@ -618,10 +623,9 @@ Zotero.Attachments = new function(){
|
|||
try {
|
||||
// Clean up
|
||||
if (itemID) {
|
||||
var destDir = Zotero.getStorageDirectory();
|
||||
destDir.append(itemID);
|
||||
if (destDir.exists()) {
|
||||
destDir.remove(true);
|
||||
var itemDir = this.getStorageDirectory(itemID);
|
||||
if (itemDir.exists()) {
|
||||
itemDir.remove(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -886,6 +890,8 @@ Zotero.Attachments = new function(){
|
|||
|
||||
/*
|
||||
* Create directory for attachment files within storage directory
|
||||
*
|
||||
* @param integer itemID Item id
|
||||
*/
|
||||
function createDirectoryForItem(itemID) {
|
||||
var dir = this.getStorageDirectory(itemID);
|
||||
|
|
@ -896,9 +902,35 @@ Zotero.Attachments = new function(){
|
|||
}
|
||||
|
||||
|
||||
/*
|
||||
* Create directory for missing attachment files within storage directory
|
||||
*
|
||||
* @param string key Item secondary lookup key
|
||||
*/
|
||||
function createDirectoryForMissingItem(key) {
|
||||
var dir = this.getMissingStorageDirectory(key);
|
||||
if (!dir.exists()) {
|
||||
dir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
|
||||
function getStorageDirectory(itemID) {
|
||||
var item = Zotero.Items.get(itemID);
|
||||
var dir = Zotero.getStorageDirectory();
|
||||
dir.append(itemID);
|
||||
dir.append(item.key);
|
||||
return dir;
|
||||
}
|
||||
|
||||
|
||||
function getMissingStorageDirectory(key) {
|
||||
if (typeof key != 'string' || !key.match(/^[A-Z0-9]{8}$/)) {
|
||||
throw ('key must be an 8-character string in '
|
||||
+ 'Zotero.Attachments.getMissingStorageDirectory()')
|
||||
}
|
||||
var dir = Zotero.getStorageDirectory();
|
||||
dir.append(key);
|
||||
return dir;
|
||||
}
|
||||
|
||||
|
|
@ -906,41 +938,11 @@ Zotero.Attachments = new function(){
|
|||
/*
|
||||
* Gets a relative descriptor for imported attachments and a persistent
|
||||
* descriptor for files outside the storage directory
|
||||
*
|
||||
* @param int missingItemID Item id to use if file is missing to
|
||||
* generate suitable path
|
||||
*/
|
||||
function getPath(file, linkMode, missingItemID) {
|
||||
var exists = file.exists();
|
||||
// TODO: can we get the itemID from the path?
|
||||
if (!missingItemID && !exists) {
|
||||
throw ('Zotero.Attachments.getPath() cannot be called on non-existent file without missingItemID');
|
||||
}
|
||||
|
||||
// If imported file doesn't exist, create one temporarily so we can get
|
||||
// the relative path (which doesn't work on non-existent files)
|
||||
if (!exists && (linkMode == self.LINK_MODE_IMPORTED_URL ||
|
||||
linkMode == self.LINK_MODE_IMPORTED_FILE)) {
|
||||
var missingFile = self.createDirectoryForItem(missingItemID);
|
||||
missingFile.append(file.leafName);
|
||||
missingFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
|
||||
|
||||
var descriptor = Zotero.Attachments.getPath(missingFile, linkMode);
|
||||
|
||||
var parentDir = missingFile.parent;
|
||||
missingFile.remove(null);
|
||||
parentDir.remove(null);
|
||||
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
file.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
|
||||
function getPath(file, linkMode) {
|
||||
if (linkMode == self.LINK_MODE_IMPORTED_URL ||
|
||||
linkMode == self.LINK_MODE_IMPORTED_FILE) {
|
||||
var storageDir = Zotero.getStorageDirectory();
|
||||
storageDir.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
return file.getRelativeDescriptor(storageDir);
|
||||
return 'storage:' + file.leafName;
|
||||
}
|
||||
|
||||
return file.persistentDescriptor;
|
||||
|
|
@ -1009,8 +1011,8 @@ Zotero.Attachments = new function(){
|
|||
|
||||
// Get path
|
||||
if (file) {
|
||||
var path = Zotero.Attachments.getPath(file, linkMode, attachmentItem.id);
|
||||
attachmentItem.attachmentPath = path;
|
||||
attachmentItem.attachmentPath
|
||||
= Zotero.Attachments.getPath(file, linkMode);
|
||||
}
|
||||
|
||||
attachmentItem.setSource(sourceItemID);
|
||||
|
|
|
|||
|
|
@ -2120,49 +2120,48 @@ Zotero.Item.prototype.getFile = function(row, skipExistsCheck) {
|
|||
}
|
||||
|
||||
if (!row) {
|
||||
var sql = "SELECT linkMode, path FROM itemAttachments WHERE itemID="
|
||||
+ this.id;
|
||||
var row = Zotero.DB.rowQuery(sql);
|
||||
var sql = "SELECT linkMode, path FROM itemAttachments WHERE itemID=?"
|
||||
var row = Zotero.DB.rowQuery(sql, this.id);
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
throw ('Attachment data not found for item ' + this.id
|
||||
+ ' in getFile()');
|
||||
throw ('Attachment data not found for item ' + this.id + ' in getFile()');
|
||||
}
|
||||
|
||||
// No associated files for linked URLs
|
||||
if (row['linkMode']==Zotero.Attachments.LINK_MODE_LINKED_URL) {
|
||||
if (row.linkMode == Zotero.Attachments.LINK_MODE_LINKED_URL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
var file = Components.classes["@mozilla.org/file/local;1"].
|
||||
createInstance(Components.interfaces.nsILocalFile);
|
||||
|
||||
if (row['linkMode']==Zotero.Attachments.LINK_MODE_IMPORTED_URL ||
|
||||
row['linkMode']==Zotero.Attachments.LINK_MODE_IMPORTED_FILE) {
|
||||
if (row.linkMode == Zotero.Attachments.LINK_MODE_IMPORTED_URL ||
|
||||
row.linkMode == Zotero.Attachments.LINK_MODE_IMPORTED_FILE) {
|
||||
try {
|
||||
var storageDir = Zotero.getStorageDirectory();
|
||||
storageDir.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
file.setRelativeDescriptor(storageDir, row['path']);
|
||||
if (row.path.indexOf("storage:") == -1) {
|
||||
Zotero.debug("Invalid attachment path '" + row.path + "'");
|
||||
throw ('Invalid path');
|
||||
}
|
||||
// Strip "storage:"
|
||||
var path = row.path.substr(8);
|
||||
var file = Zotero.Attachments.getStorageDirectory(this.id);
|
||||
file.append(path);
|
||||
if (!file.exists()) {
|
||||
throw('Invalid relative descriptor');
|
||||
Zotero.debug("Attachment file '" + path + "' not found");
|
||||
throw ('File not found');
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
// See if this is a persistent path
|
||||
// (deprecated for imported attachments)
|
||||
Zotero.debug('Invalid relative descriptor -- trying persistent');
|
||||
Zotero.debug('Trying as persistent descriptor');
|
||||
try {
|
||||
file.persistentDescriptor = row['path'];
|
||||
|
||||
var storageDir = Zotero.getStorageDirectory();
|
||||
storageDir.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
var path = file.getRelativeDescriptor(storageDir);
|
||||
var file = Components.classes["@mozilla.org/file/local;1"].
|
||||
createInstance(Components.interfaces.nsILocalFile);
|
||||
file.persistentDescriptor = row.path;
|
||||
|
||||
// If valid, convert this to a relative descriptor
|
||||
if (file.exists()) {
|
||||
Zotero.DB.query("UPDATE itemAttachments SET path=? WHERE itemID=?",
|
||||
[path, this.id]);
|
||||
["storage:" + file.leafName, this.id]);
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
|
|
@ -2171,16 +2170,19 @@ Zotero.Item.prototype.getFile = function(row, skipExistsCheck) {
|
|||
}
|
||||
}
|
||||
else {
|
||||
var file = Components.classes["@mozilla.org/file/local;1"].
|
||||
createInstance(Components.interfaces.nsILocalFile);
|
||||
|
||||
try {
|
||||
file.persistentDescriptor = row['path'];
|
||||
file.persistentDescriptor = row.path;
|
||||
}
|
||||
catch (e) {
|
||||
// See if this is an old relative path (deprecated)
|
||||
Zotero.debug('Invalid persistent descriptor -- trying relative');
|
||||
try {
|
||||
var refDir = (row['linkMode']==this.LINK_MODE_LINKED_FILE)
|
||||
var refDir = (row.linkMode == this.LINK_MODE_LINKED_FILE)
|
||||
? Zotero.getZoteroDirectory() : Zotero.getStorageDirectory();
|
||||
file.setRelativeDescriptor(refDir, row['path']);
|
||||
file.setRelativeDescriptor(refDir, row.path);
|
||||
// If valid, convert this to a persistent descriptor
|
||||
if (file.exists()) {
|
||||
Zotero.DB.query("UPDATE itemAttachments SET path=? WHERE itemID=?",
|
||||
|
|
@ -2229,7 +2231,7 @@ Zotero.Item.prototype.renameAttachmentFile = function(newName, overwrite) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
file.moveTo(file.parent, newName);
|
||||
file.moveTo(null, newName);
|
||||
this.relinkAttachmentFile(file);
|
||||
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ Zotero.Fulltext = new function(){
|
|||
}
|
||||
|
||||
var item = Zotero.Items.get(itemID);
|
||||
var linkMode = item.getAttachmentLinkMode();
|
||||
var linkMode = item.attachmentLinkMode;
|
||||
// If file is stored outside of Zotero, create a directory for the item
|
||||
// in the storage directory and save the cache file there
|
||||
if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE) {
|
||||
|
|
@ -644,8 +644,7 @@ Zotero.Fulltext = new function(){
|
|||
* Gets the number of pages from the PDF info cache file
|
||||
*/
|
||||
function getTotalPagesFromFile(itemID) {
|
||||
var item = Zotero.Items.get(itemID);
|
||||
var file = Zotero.Attachments.getStorageDirectory(item.getID());
|
||||
var file = Zotero.Attachments.getStorageDirectory(itemID);
|
||||
file.append(this.pdfInfoCacheFile);
|
||||
if (!file.exists()) {
|
||||
return false;
|
||||
|
|
@ -1029,8 +1028,7 @@ Zotero.Fulltext = new function(){
|
|||
|
||||
|
||||
function _getItemCacheFile(itemID) {
|
||||
var cacheFile = Zotero.getStorageDirectory();
|
||||
cacheFile.append(itemID);
|
||||
var cacheFile = Zotero.Attachments.getStorageDirectory(itemID);
|
||||
cacheFile.append(self.pdfConverterCacheFile);
|
||||
return cacheFile;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1531,6 +1531,61 @@ Zotero.Schema = new function(){
|
|||
}
|
||||
}
|
||||
statement.reset();
|
||||
|
||||
// Migrate attachment folders to secondary keys
|
||||
Zotero.DB.query("UPDATE itemAttachments SET path=REPLACE(path, itemID || '/', 'storage:') WHERE path REGEXP '^[0-9]+/'");
|
||||
|
||||
if (Zotero.Prefs.get('useDataDir')) {
|
||||
var dataDir = Components.classes["@mozilla.org/file/local;1"].createInstance(Components.interfaces.nsILocalFile);
|
||||
dataDir.persistentDescriptor = Zotero.Prefs.get('dataDir');
|
||||
}
|
||||
else {
|
||||
var dataDir = Zotero.getProfileDirectory();
|
||||
dataDir.append('zotero');
|
||||
}
|
||||
if (!dataDir.exists() || !dataDir.isDirectory()){
|
||||
var e = { name: "NS_ERROR_FILE_NOT_FOUND" };
|
||||
throw (e);
|
||||
}
|
||||
var movedFiles37 = {};
|
||||
var moveReport = '';
|
||||
var orphaned = dataDir.clone();
|
||||
var storage37 = dataDir.clone();
|
||||
var moveReportFile = dataDir.clone();
|
||||
orphaned.append('orphaned-files');
|
||||
storage37.append('storage');
|
||||
moveReportFile.append('zotero.moved-files.' + fromVersion + '.bak');
|
||||
var keys = {};
|
||||
var rows = Zotero.DB.query("SELECT itemID, key FROM items");
|
||||
for each(var row in rows) {
|
||||
keys[row.itemID] = row.key;
|
||||
}
|
||||
var entries = storage37.directoryEntries;
|
||||
while (entries.hasMoreElements()) {
|
||||
var file = entries.getNext();
|
||||
file.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
var id = parseInt(file.leafName);
|
||||
if (!file.isDirectory() || isNaN(id)) {
|
||||
continue;
|
||||
}
|
||||
if (keys[id]) {
|
||||
file.moveTo(null, keys[id]);
|
||||
moveReport += keys[id] + ' ' + id + "\n";
|
||||
}
|
||||
else {
|
||||
if (!orphaned.exists()) {
|
||||
orphaned.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
|
||||
}
|
||||
file.moveTo(orphaned, null);
|
||||
}
|
||||
movedFiles37[id] = file;
|
||||
}
|
||||
if (moveReport) {
|
||||
moveReport = 'The following directory names in storage were changed:\n'
|
||||
+ '------------------------------------------------------\n'
|
||||
+ moveReport;
|
||||
Zotero.File.putContents(moveReportFile, moveReport);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1538,7 +1593,15 @@ Zotero.Schema = new function(){
|
|||
|
||||
Zotero.DB.commitTransaction();
|
||||
}
|
||||
catch(e){
|
||||
catch (e) {
|
||||
if (movedFiles37) {
|
||||
for (var id in movedFiles37) {
|
||||
try {
|
||||
movedFiles37[id].moveTo(storage37, id);
|
||||
}
|
||||
catch (e2) { Zotero.debug(e2); }
|
||||
}
|
||||
}
|
||||
Zotero.DB.rollbackTransaction();
|
||||
throw(e);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue