zotero/test/tests/zfsTest.js
2016-03-07 20:12:48 -05:00

778 lines
24 KiB
JavaScript

"use strict";
describe("Zotero.Sync.Storage.Mode.ZFS", function () {
//
// Setup
//
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var win, server, requestCount;
var responses = {};
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function resetRequestCount() {
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
}
function assertRequestCount(count) {
assert.equal(
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
count
);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs');
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
resetRequestCount();
})
var setup = Zotero.Promise.coroutine(function* (options = {}) {
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
controller: new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client,
maxS3ConsecutiveFailures: 2
}),
stopOnError: true
});
return { engine, client, caller };
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("Syncing", function () {
it("should skip downloads if not marked as needed", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
var result = yield engine.start();
assertRequestCount(0);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, new File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
itemJSON.data.mtime = yield item.attachmentModificationTime;
itemJSON.data.md5 = yield item.attachmentHash;
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id),
(yield item.attachmentHash)
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})