dc12a2c95a
We weren't making actual upload requests after a quota error if the file would exceed the quota, but we were still going through all attachments to upload, which in some cases involves making stat() calls. We now just stop the queue immediately after a quota error or when starting a new background sync after a previous quota error. Closes #1255
1159 lines
35 KiB
JavaScript
1159 lines
35 KiB
JavaScript
"use strict";
|
|
|
|
describe("Zotero.Sync.Storage.Mode.ZFS", function () {
|
|
//
|
|
// Setup
|
|
//
|
|
Components.utils.import("resource://zotero-unit/httpd.js");
|
|
|
|
var apiKey = Zotero.Utilities.randomString(24);
|
|
var port = 16213;
|
|
var baseURL = `http://localhost:${port}/`;
|
|
|
|
var win, server, requestCount;
|
|
var responses = {};
|
|
|
|
function setResponse(response) {
|
|
setHTTPResponse(server, baseURL, response, responses);
|
|
}
|
|
|
|
function resetRequestCount() {
|
|
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
|
|
}
|
|
|
|
function assertRequestCount(count) {
|
|
assert.equal(
|
|
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
|
|
count
|
|
);
|
|
}
|
|
|
|
function parseQueryString(str) {
|
|
var queryStringParams = str.split('&');
|
|
var params = {};
|
|
for (let param of queryStringParams) {
|
|
let [ key, val ] = param.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
return params;
|
|
}
|
|
|
|
function assertAPIKey(request) {
|
|
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
|
|
}
|
|
|
|
//
|
|
// Tests
|
|
//
|
|
beforeEach(function* () {
|
|
yield resetDB({
|
|
thisArg: this,
|
|
skipBundledFiles: true
|
|
});
|
|
win = yield loadZoteroPane();
|
|
|
|
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
|
|
server = sinon.fakeServer.create();
|
|
server.autoRespond = true;
|
|
|
|
this.httpd = new HttpServer();
|
|
this.httpd.start(port);
|
|
|
|
yield Zotero.Users.setCurrentUserID(1);
|
|
yield Zotero.Users.setCurrentUsername("testuser");
|
|
|
|
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs');
|
|
|
|
// Set download-on-sync by default
|
|
Zotero.Sync.Storage.Local.downloadOnSync(
|
|
Zotero.Libraries.userLibraryID, true
|
|
);
|
|
|
|
resetRequestCount();
|
|
})
|
|
|
|
var setup = Zotero.Promise.coroutine(function* (options = {}) {
|
|
Components.utils.import("resource://zotero/concurrentCaller.js");
|
|
var stopOnError = options.stopOnError !== undefined ? options.stopOnError : true;
|
|
var caller = new ConcurrentCaller(1);
|
|
caller.setLogger(msg => Zotero.debug(msg));
|
|
caller.stopOnError = stopOnError;
|
|
|
|
Components.utils.import("resource://zotero/config.js");
|
|
var client = new Zotero.Sync.APIClient({
|
|
baseURL,
|
|
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
|
|
apiKey,
|
|
caller,
|
|
background: options.background || true
|
|
});
|
|
|
|
var engine = new Zotero.Sync.Storage.Engine({
|
|
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
|
|
controller: new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client,
|
|
maxS3ConsecutiveFailures: 2
|
|
}),
|
|
background: options.background,
|
|
stopOnError
|
|
});
|
|
|
|
return { engine, client, caller };
|
|
})
|
|
|
|
afterEach(function* () {
|
|
var defer = new Zotero.Promise.defer();
|
|
this.httpd.stop(() => defer.resolve());
|
|
yield defer.promise;
|
|
win.close();
|
|
})
|
|
|
|
after(function* () {
|
|
this.timeout(60000);
|
|
//yield resetDB();
|
|
win.close();
|
|
})
|
|
|
|
|
|
describe("Syncing", function () {
|
|
it("should skip downloads if not marked as needed", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
|
|
var result = yield engine.start();
|
|
|
|
assertRequestCount(0);
|
|
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
})
|
|
|
|
it("should ignore download for a remotely missing file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
yield item.saveTx();
|
|
|
|
this.httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 404, null);
|
|
}
|
|
}
|
|
);
|
|
var result = yield engine.start();
|
|
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
assert.isFalse(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
})
|
|
|
|
it("shouldn't update storageVersion if stopped", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var items = [];
|
|
for (let i = 0; i < 5; i++) {
|
|
let item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
yield item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
var call = 0;
|
|
var stub = sinon.stub(engine.controller, 'downloadFile').callsFake(function () {
|
|
call++;
|
|
if (call == 1) {
|
|
engine.stop();
|
|
}
|
|
return new Zotero.Sync.Storage.Result;
|
|
});
|
|
|
|
var result = yield engine.start();
|
|
|
|
stub.restore();
|
|
|
|
assert.equal(library.storageVersion, 0);
|
|
});
|
|
|
|
it("should handle a remotely failing file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
yield item.saveTx();
|
|
|
|
this.httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 500, null);
|
|
}
|
|
}
|
|
);
|
|
// TODO: In stopOnError mode, this the promise is rejected.
|
|
// This should probably test with stopOnError mode turned off instead.
|
|
var e = yield getPromiseError(engine.start());
|
|
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
|
|
|
|
assert.isTrue(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, 0);
|
|
})
|
|
|
|
it("should download a missing file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
// TODO: Test binary data
|
|
var text = Zotero.Utilities.randomString();
|
|
item.attachmentSyncState = "to_download";
|
|
yield item.saveTx();
|
|
|
|
var mtime = "1441252524905";
|
|
var md5 = Zotero.Utilities.Internal.md5(text)
|
|
|
|
var s3Path = `pretend-s3/${item.key}`;
|
|
this.httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
if (!request.hasHeader('Zotero-API-Key')) {
|
|
response.setStatusLine(null, 403, "Forbidden");
|
|
return;
|
|
}
|
|
var key = request.getHeader('Zotero-API-Key');
|
|
if (key != apiKey) {
|
|
response.setStatusLine(null, 403, "Invalid key");
|
|
return;
|
|
}
|
|
response.setStatusLine(null, 302, "Found");
|
|
response.setHeader("Zotero-File-Modification-Time", mtime, false);
|
|
response.setHeader("Zotero-File-MD5", md5, false);
|
|
response.setHeader("Zotero-File-Compressed", "No", false);
|
|
response.setHeader("Location", baseURL + s3Path, false);
|
|
}
|
|
}
|
|
);
|
|
this.httpd.registerPathHandler(
|
|
"/" + s3Path,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 200, "OK");
|
|
response.write(text);
|
|
}
|
|
}
|
|
);
|
|
var result = yield engine.start();
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
|
|
assert.equal(contents, text);
|
|
|
|
assert.isFalse(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
})
|
|
|
|
it("should upload new files", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
// Single file
|
|
var file1 = getTestDataDirectory();
|
|
file1.append('test.png');
|
|
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
|
|
var mtime1 = yield item1.attachmentModificationTime;
|
|
var hash1 = yield item1.attachmentHash;
|
|
var path1 = item1.getFilePath();
|
|
var filename1 = 'test.png';
|
|
var size1 = (yield OS.File.stat(path1)).size;
|
|
var contentType1 = 'image/png';
|
|
var prefix1 = Zotero.Utilities.randomString();
|
|
var suffix1 = Zotero.Utilities.randomString();
|
|
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
let file1Blob = File.createFromFileName ? File.createFromFileName(file1.path) : new File(file1);
|
|
if (file1Blob.then) {
|
|
file1Blob = yield file1Blob;
|
|
}
|
|
|
|
// HTML file with auxiliary image
|
|
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
|
|
var parentItem = yield createDataObject('item');
|
|
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
|
|
file: file2,
|
|
url: 'http://example.com/',
|
|
parentItemID: parentItem.id,
|
|
title: 'Test',
|
|
contentType: 'text/html',
|
|
charset: 'utf-8'
|
|
});
|
|
var mtime2 = yield item2.attachmentModificationTime;
|
|
var hash2 = yield item2.attachmentHash;
|
|
var path2 = item2.getFilePath();
|
|
var filename2 = 'index.html';
|
|
var size2 = (yield OS.File.stat(path2)).size;
|
|
var contentType2 = 'text/html';
|
|
var charset2 = 'utf-8';
|
|
var prefix2 = Zotero.Utilities.randomString();
|
|
var suffix2 = Zotero.Utilities.randomString();
|
|
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
var deferreds = [];
|
|
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Get upload authorization for single file
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.md5, hash1);
|
|
assert.equal(params.mtime, mtime1);
|
|
assert.equal(params.filename, filename1);
|
|
assert.equal(params.filesize, size1);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/1",
|
|
contentType: contentType1,
|
|
prefix: prefix1,
|
|
suffix: suffix1,
|
|
uploadKey: uploadKey1
|
|
})
|
|
);
|
|
}
|
|
// Get upload authorization for multi-file zip
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
// Verify ZIP hash
|
|
let tmpZipPath = OS.Path.join(
|
|
Zotero.getTempDirectory().path,
|
|
item2.key + '.zip'
|
|
);
|
|
deferreds.push({
|
|
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
|
|
.then(function (md5) {
|
|
assert.equal(params.zipMD5, md5);
|
|
})
|
|
});
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
Zotero.debug(params);
|
|
assert.equal(params.md5, hash2);
|
|
assert.notEqual(params.zipMD5, hash2);
|
|
assert.equal(params.mtime, mtime2);
|
|
assert.equal(params.filename, filename2);
|
|
assert.equal(params.zipFilename, item2.key + ".zip");
|
|
assert.isTrue(parseInt(params.filesize) == params.filesize);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/2",
|
|
contentType: 'application/zip',
|
|
prefix: prefix2,
|
|
suffix: suffix2,
|
|
uploadKey: uploadKey2
|
|
})
|
|
);
|
|
}
|
|
// Upload single file to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
|
|
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
|
|
assert.equal(
|
|
req.requestBody.size,
|
|
(new Blob(
|
|
[
|
|
prefix1,
|
|
file1Blob,
|
|
suffix1
|
|
]
|
|
).size)
|
|
);
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Upload multi-file ZIP to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
|
|
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
|
|
|
|
// Verify uploaded ZIP file
|
|
let tmpZipPath = OS.Path.join(
|
|
Zotero.getTempDirectory().path,
|
|
Zotero.Utilities.randomString() + '.zip'
|
|
);
|
|
|
|
let deferred = Zotero.Promise.defer();
|
|
deferreds.push(deferred);
|
|
var reader = new FileReader();
|
|
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
|
|
try {
|
|
|
|
let file = yield OS.File.open(tmpZipPath, {
|
|
create: true
|
|
});
|
|
|
|
var contents = new Uint8Array(reader.result);
|
|
contents = contents.slice(prefix2.length, suffix2.length * -1);
|
|
yield file.write(contents);
|
|
yield file.close();
|
|
|
|
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
|
|
.createInstance(Components.interfaces.nsIZipReader);
|
|
zr.open(Zotero.File.pathToFile(tmpZipPath));
|
|
zr.test(null);
|
|
var entries = zr.findEntries('*');
|
|
var entryNames = [];
|
|
while (entries.hasMore()) {
|
|
entryNames.push(entries.getNext());
|
|
}
|
|
assert.equal(entryNames.length, 2);
|
|
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
|
|
assert.equal(zr.getEntry('index.html').realSize, size2);
|
|
assert.equal(zr.getEntry('img.gif').realSize, 42);
|
|
|
|
deferred.resolve();
|
|
}
|
|
catch (e) {
|
|
deferred.reject(e);
|
|
}
|
|
}));
|
|
reader.readAsArrayBuffer(req.requestBody);
|
|
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Register single-file upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
|
|
&& req.requestBody.indexOf('upload=') != -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.upload, uploadKey1);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 10
|
|
},
|
|
""
|
|
);
|
|
}
|
|
// Register multi-file upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
|
|
&& req.requestBody.indexOf('upload=') != -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.upload, uploadKey2);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 15
|
|
},
|
|
""
|
|
);
|
|
}
|
|
})
|
|
|
|
// TODO: One-step uploads
|
|
/*// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/json" + fixSinonBug
|
|
);
|
|
|
|
let params = JSON.parse(req.requestBody);
|
|
assert.equal(params.md5, hash);
|
|
assert.equal(params.mtime, mtime);
|
|
assert.equal(params.filename, filename);
|
|
assert.equal(params.size, size);
|
|
assert.equal(params.contentType, contentType);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3",
|
|
headers: {
|
|
"Content-Type": contentType,
|
|
"Content-MD5": hash,
|
|
//"Content-Length": params.size, process but don't return
|
|
//"x-amz-meta-"
|
|
},
|
|
uploadKey
|
|
})
|
|
);
|
|
}
|
|
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
|
|
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
|
|
assert.instanceOf(req.requestBody, File);
|
|
req.respond(201, {}, "");
|
|
}
|
|
})*/
|
|
var result = yield engine.start();
|
|
|
|
yield Zotero.Promise.all(deferreds.map(d => d.promise));
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isTrue(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
// Check local objects
|
|
assert.equal(item1.attachmentSyncedModificationTime, mtime1);
|
|
assert.equal(item1.attachmentSyncedHash, hash1);
|
|
assert.equal(item1.version, 10);
|
|
assert.equal(item2.attachmentSyncedModificationTime, mtime2);
|
|
assert.equal(item2.attachmentSyncedHash, hash2);
|
|
assert.equal(item2.version, 15);
|
|
})
|
|
|
|
it("should update local info for remotely updated file that matches local file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
yield library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.txt');
|
|
var item = yield Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.attachmentSyncState = "to_download";
|
|
yield item.saveTx();
|
|
var path = yield item.getFilePathAsync();
|
|
yield OS.File.setDates(path, null, new Date() - 100000);
|
|
|
|
var json = item.toJSON();
|
|
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
|
|
|
|
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) + "";
|
|
var md5 = Zotero.Utilities.Internal.md5(file)
|
|
|
|
var s3Path = `pretend-s3/${item.key}`;
|
|
this.httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
if (!request.hasHeader('Zotero-API-Key')) {
|
|
response.setStatusLine(null, 403, "Forbidden");
|
|
return;
|
|
}
|
|
var key = request.getHeader('Zotero-API-Key');
|
|
if (key != apiKey) {
|
|
response.setStatusLine(null, 403, "Invalid key");
|
|
return;
|
|
}
|
|
response.setStatusLine(null, 302, "Found");
|
|
response.setHeader("Zotero-File-Modification-Time", mtime, false);
|
|
response.setHeader("Zotero-File-MD5", md5, false);
|
|
response.setHeader("Zotero-File-Compressed", "No", false);
|
|
response.setHeader("Location", baseURL + s3Path, false);
|
|
}
|
|
}
|
|
);
|
|
var result = yield engine.start();
|
|
|
|
assert.equal(item.attachmentSyncedModificationTime, mtime);
|
|
yield assert.eventually.equal(item.attachmentModificationTime, mtime);
|
|
assert.isTrue(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
})
|
|
|
|
it("should update local info for file that already exists on the server", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file: file });
|
|
item.version = 5;
|
|
yield item.saveTx();
|
|
var json = item.toJSON();
|
|
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
|
|
|
|
var mtime = yield item.attachmentModificationTime;
|
|
var hash = yield item.attachmentHash;
|
|
var path = item.getFilePath();
|
|
var filename = 'test.png';
|
|
var size = (yield OS.File.stat(path)).size;
|
|
var contentType = 'image/png';
|
|
|
|
var newVersion = 10;
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Get upload authorization for single file
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": newVersion
|
|
},
|
|
JSON.stringify({
|
|
exists: 1,
|
|
})
|
|
);
|
|
}
|
|
})
|
|
|
|
// TODO: One-step uploads
|
|
var result = yield engine.start();
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isTrue(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
// Check local objects
|
|
assert.equal(item.attachmentSyncedModificationTime, mtime);
|
|
assert.equal(item.attachmentSyncedHash, hash);
|
|
assert.equal(item.version, newVersion);
|
|
})
|
|
|
|
|
|
it("should retry with If-None-Match on 412 with missing remote hash", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
item.attachmentSyncedModificationTime = Date.now();
|
|
item.attachmentSyncedHash = 'bd4c33e03798a7e8bc0b46f8bda74fac'
|
|
yield item.saveTx();
|
|
|
|
var contentType = 'image/png';
|
|
var prefix = Zotero.Utilities.randomString();
|
|
var suffix = Zotero.Utilities.randomString();
|
|
var uploadKey = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
var called = 0;
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Try with If-Match
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')
|
|
&& req.requestHeaders["If-Match"] == item.attachmentSyncedHash) {
|
|
called++;
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
"If-Match set but file does not exist"
|
|
);
|
|
}
|
|
// Retry with If-None-Match
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
assert.equal(called++, 1);
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/1",
|
|
contentType: contentType,
|
|
prefix: prefix,
|
|
suffix: suffix,
|
|
uploadKey: uploadKey
|
|
})
|
|
);
|
|
}
|
|
// Upload file to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
|
|
assert.equal(called++, 2);
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Use If-None-Match when registering upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.includes('upload=')) {
|
|
assert.equal(called++, 3);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 10
|
|
},
|
|
""
|
|
);
|
|
}
|
|
});
|
|
|
|
var result = yield engine.start();
|
|
assert.equal(called, 4);
|
|
});
|
|
|
|
|
|
it("should stop uploading files on quota error", async function () {
|
|
var { engine, client, caller } = await setup({ stopOnError: false });
|
|
|
|
var numItems = 4;
|
|
var items = [];
|
|
for (let i = 0; i < numItems; i++) {
|
|
let item = await importFileAttachment('test.png');
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
var requests = 0;
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url.startsWith(`${baseURL}users/1/items/`)
|
|
&& req.url.endsWith('/file')
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
requests++;
|
|
req.respond(
|
|
413,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10,
|
|
"Zotero-Storage-Usage": "300",
|
|
"Zotero-Storage-Quota": "300"
|
|
},
|
|
"File would exceed quota (299.7 + 0.5 > 300)"
|
|
);
|
|
}
|
|
})
|
|
|
|
await engine.start();
|
|
assert.equal(requests, Zotero.Prefs.get('sync.storage.maxUploads'));
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.delete(items[0].libraryID);
|
|
});
|
|
|
|
|
|
// If there was a quota error in a previous run and remaining storage was determined to be
|
|
// very low, stop further file uploads for a background sync even when we bail without an
|
|
// HTTP request. A manual sync clears the remaining-storage value.
|
|
it("should stop uploading files for background sync if no storage remaining after previous quota error", async function () {
|
|
var { engine, client, caller } = await setup({ background: true, stopOnError: false });
|
|
|
|
var numItems = 4;
|
|
var items = [];
|
|
for (let i = 0; i < numItems; i++) {
|
|
let item = await importFileAttachment('test.png');
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.set(items[0].libraryID, 0);
|
|
|
|
var spy = sinon.spy(engine.controller, 'uploadFile');
|
|
await engine.start()
|
|
|
|
assert.equal(spy.callCount, Zotero.Prefs.get('sync.storage.maxUploads'));
|
|
spy.restore();
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.delete(items[0].libraryID);
|
|
});
|
|
})
|
|
|
|
|
|
describe("#uploadFile()", function () {
|
|
it("should compress single-file HTML snapshots", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var dir = await getTempDirectory();
|
|
var file = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
|
|
var file2 = OS.Path.join(dir, 'index.html');
|
|
await OS.File.copy(file, file2);
|
|
file = file2;
|
|
|
|
var parentItem = await createDataObject('item');
|
|
var item = await Zotero.Attachments.importSnapshotFromFile({
|
|
file,
|
|
url: 'http://example.com/',
|
|
parentItemID: parentItem.id,
|
|
title: 'Test',
|
|
contentType: 'text/html',
|
|
charset: 'utf-8'
|
|
});
|
|
|
|
var request = { name: item.libraryKey };
|
|
|
|
var stub = sinon.stub(zfs, '_processUploadFile').returns(Zotero.Promise.resolve());
|
|
await zfs.uploadFile(request);
|
|
|
|
var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
|
|
// _getUploadFile() should return the ZIP file
|
|
assert.equal((await zfs._getUploadFile(item)).path, zipFile);
|
|
assert.isTrue(await OS.File.exists(zipFile));
|
|
|
|
stub.restore();
|
|
});
|
|
});
|
|
|
|
|
|
describe("#_processUploadFile()", function () {
|
|
it("should handle 404 from upload authorization request", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
yield item.saveTx();
|
|
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.mtime = yield item.attachmentModificationTime;
|
|
itemJSON.data.md5 = yield item.attachmentHash;
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')) {
|
|
req.respond(
|
|
404,
|
|
{
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"Not Found"
|
|
);
|
|
}
|
|
})
|
|
|
|
var result = yield zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.isTrue(result.syncRequired);
|
|
});
|
|
|
|
it("should handle 412 with matching version and hash matching local file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
yield item.saveTx();
|
|
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.mtime = yield item.attachmentModificationTime;
|
|
itemJSON.data.md5 = yield item.attachmentHash;
|
|
|
|
// Set saved hash to a different value, which should be overwritten
|
|
//
|
|
// We're also testing cases where a hash isn't set for a file (e.g., if the
|
|
// storage directory was transferred, the mtime doesn't match, but the file was
|
|
// never downloaded), but there's no difference in behavior
|
|
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
item.attachmentSyncedHash = dbHash;
|
|
yield item.saveTx({ skipAll: true });
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-Match"] == dbHash) {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"ETag does not match current version of file"
|
|
);
|
|
}
|
|
})
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items?itemKey=${item.key}&includeTrashed=1`,
|
|
status: 200,
|
|
text: JSON.stringify([itemJSON])
|
|
});
|
|
|
|
var result = yield zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.equal(item.attachmentSyncedHash, (yield item.attachmentHash));
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
assert.isFalse(result.fileSyncRequired);
|
|
})
|
|
|
|
it("should handle 412 with matching version and hash not matching local file", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
yield item.saveTx();
|
|
|
|
var fileHash = yield item.attachmentHash;
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"If-None-Match: * set but file exists"
|
|
);
|
|
}
|
|
})
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items?itemKey=${item.key}&includeTrashed=1`,
|
|
status: 200,
|
|
text: JSON.stringify([itemJSON])
|
|
});
|
|
|
|
var result = yield zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.isNull(item.attachmentSyncedHash);
|
|
assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT);
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
assert.isTrue(result.fileSyncRequired);
|
|
})
|
|
|
|
it("should handle 412 with greater version", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
yield item.saveTx();
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10
|
|
},
|
|
"If-None-Match: * set but file exists"
|
|
);
|
|
}
|
|
})
|
|
|
|
var result = yield zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.equal(item.version, 5);
|
|
assert.equal(item.synced, true);
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isTrue(result.syncRequired);
|
|
// Item should be marked for redownloading
|
|
var versions = yield Zotero.Sync.Data.Local.getObjectsToTryFromSyncQueue('item', item.libraryID);
|
|
assert.include(versions, item.key);
|
|
});
|
|
|
|
|
|
it("should handle 413 on quota limit", function* () {
|
|
var { engine, client, caller } = yield setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = yield Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
yield item.saveTx();
|
|
|
|
var responses = 0;
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
responses++;
|
|
req.respond(
|
|
413,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10,
|
|
"Zotero-Storage-Usage": "300",
|
|
"Zotero-Storage-Quota": "300"
|
|
},
|
|
"File would exceed quota (299.7 + 0.5 > 300)"
|
|
);
|
|
}
|
|
})
|
|
|
|
var e = yield getPromiseError(zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
}));
|
|
assert.ok(e);
|
|
assert.equal(e.errorType, 'warning');
|
|
assert.include(e.message, 'test.png');
|
|
assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings'));
|
|
assert.equal(responses, 1);
|
|
|
|
// Try again
|
|
var e = yield getPromiseError(zfs.uploadFile({
|
|
name: item.libraryKey,
|
|
engine
|
|
}));
|
|
assert.ok(e);
|
|
assert.equal(e.errorType, 'warning');
|
|
assert.include(e.message, 'test.png');
|
|
assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings'));
|
|
// Shouldn't have been another request. A manual sync resets the flag, but we're not
|
|
// testing that here.
|
|
assert.equal(responses, 1);
|
|
})
|
|
})
|
|
})
|