mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Gzip-compress API uploads larger than 1000 characters

Browse source
This commit is contained in:
Dan Stillman 2016-03-28 02:35:27 -04:00
parent 144d02e36c
commit 35530af1fb
8 changed files with 3261 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
chrome/content/zotero/xpcom/http.js
View file

@ -181,9 +181,25 @@ Zotero.HTTP = new function() {
}
// Send headers
var headers = (options && options.headers) || {};
if (options.body && !headers["Content-Type"]) {
headers["Content-Type"] = "application/x-www-form-urlencoded";
var headers = {};
if (options && options.headers) {
Object.assign(headers, options.headers);
}
var compressedBody = false;
if (options.body) {
if (!headers["Content-Type"]) {
headers["Content-Type"] = "application/x-www-form-urlencoded";
}
if (options.compressBody && this.isWriteMethod(method)) {
headers['Content-Encoding'] = 'gzip';
compressedBody = yield Zotero.Utilities.Internal.gzip(options.body);
let oldLen = options.body.length;
let newLen = compressedBody.length;
Zotero.debug(`${method} body gzipped from ${oldLen} to ${newLen}; `
+ Math.round(((oldLen - newLen) / oldLen) * 100) + "% savings");
}
}
if (options.debug) {
if (headers["Zotero-API-Key"]) {
@ -248,7 +264,19 @@ Zotero.HTTP = new function() {
options.cookieSandbox.attachToInterfaceRequestor(xmlhttp);
}
xmlhttp.send(options.body || null);
// Send binary data
if (compressedBody) {
let numBytes = compressedBody.length;
let ui8Data = new Uint8Array(numBytes);
for (let i = 0; i < numBytes; i++) {
ui8Data[i] = compressedBody.charCodeAt(i) & 0xff;
}
xmlhttp.send(ui8Data);
}
// Send regular request
else {
xmlhttp.send(options.body || null);
}
return deferred.promise;
});
@ -705,6 +733,11 @@ Zotero.HTTP = new function() {
}
this.isWriteMethod = function (method) {
return method == 'POST' || method == 'PUT' || method == 'PATCH';
};
this.getDisplayURI = function (uri) {
var disp = uri.clone();
if (disp.password) {

5
chrome/content/zotero/xpcom/sync/syncAPIClient.js
View file

@ -43,6 +43,7 @@ Zotero.Sync.APIClient = function (options) {
Zotero.Sync.APIClient.prototype = {
MAX_OBJECTS_PER_REQUEST: 100,
MIN_GZIP_SIZE: 1000,
getKeyInfo: Zotero.Promise.coroutine(function* (options={}) {
@ -571,6 +572,10 @@ Zotero.Sync.APIClient.prototype = {
opts.dontCache = true;
opts.foreground = !options.background;
opts.responseType = options.responseType || 'text';
if (options.body && options.body.length >= this.MIN_GZIP_SIZE) {
opts.compressBody = true;
}
var tries = 0;
var failureDelayGenerator = null;
while (true) {

119
chrome/content/zotero/xpcom/utilities_internal.js
View file

@ -225,6 +225,125 @@ Zotero.Utilities.Internal = {
},
gzip: Zotero.Promise.coroutine(function* (data) {
var deferred = Zotero.Promise.defer();
// Get input stream from POST data
var unicodeConverter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"]
.createInstance(Components.interfaces.nsIScriptableUnicodeConverter);
unicodeConverter.charset = "UTF-8";
var is = unicodeConverter.convertToInputStream(data);
// Initialize stream converter
var converter = Components.classes["@mozilla.org/streamconv;1?from=uncompressed&to=gzip"]
.createInstance(Components.interfaces.nsIStreamConverter);
converter.asyncConvertData(
"uncompressed",
"gzip",
{
binaryInputStream: null,
size: 0,
data: '',
onStartRequest: function (request, context) {},
onStopRequest: function (request, context, status) {
this.binaryInputStream.close();
delete this.binaryInputStream;
deferred.resolve(this.data);
},
onDataAvailable: function (request, context, inputStream, offset, count) {
this.size += count;
this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
.createInstance(Components.interfaces.nsIBinaryInputStream)
this.binaryInputStream.setInputStream(inputStream);
this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
},
QueryInterface: function (iid) {
if (iid.equals(Components.interfaces.nsISupports)
|| iid.equals(Components.interfaces.nsIStreamListener)) {
return this;
}
throw Components.results.NS_ERROR_NO_INTERFACE;
}
},
null
);
// Send input stream to stream converter
var pump = Components.classes["@mozilla.org/network/input-stream-pump;1"]
.createInstance(Components.interfaces.nsIInputStreamPump);
pump.init(is, -1, -1, 0, 0, true);
pump.asyncRead(converter, null);
return deferred.promise;
}),
gunzip: Zotero.Promise.coroutine(function* (data) {
var deferred = Zotero.Promise.defer();
Components.utils.import("resource://gre/modules/NetUtil.jsm");
var is = Components.classes["@mozilla.org/io/string-input-stream;1"]
.createInstance(Ci.nsIStringInputStream);
is.setData(data, data.length);
var bis = Components.classes["@mozilla.org/binaryinputstream;1"]
.createInstance(Components.interfaces.nsIBinaryInputStream);
bis.setInputStream(is);
// Initialize stream converter
var converter = Components.classes["@mozilla.org/streamconv;1?from=gzip&to=uncompressed"]
.createInstance(Components.interfaces.nsIStreamConverter);
converter.asyncConvertData(
"gzip",
"uncompressed",
{
data: '',
onStartRequest: function (request, context) {},
onStopRequest: function (request, context, status) {
deferred.resolve(this.data);
},
onDataAvailable: function (request, context, inputStream, offset, count) {
this.data += NetUtil.readInputStreamToString(
inputStream,
inputStream.available(),
{
charset: 'UTF-8',
replacement: 65533
}
)
},
QueryInterface: function (iid) {
if (iid.equals(Components.interfaces.nsISupports)
|| iid.equals(Components.interfaces.nsIStreamListener)) {
return this;
}
throw Components.results.NS_ERROR_NO_INTERFACE;
}
},
null
);
// Send input stream to stream converter
var pump = Components.classes["@mozilla.org/network/input-stream-pump;1"]
.createInstance(Components.interfaces.nsIInputStreamPump);
pump.init(bis, -1, -1, 0, 0, true);
pump.asyncRead(converter, null);
return deferred.promise;
}),
/**
* Unicode normalization
*/

1
test/content/runtests.html
View file

@ -11,6 +11,7 @@
<script src="resource://zotero-unit/mocha/mocha.js"></script>
<script src="resource://zotero-unit/sinon.js"></script>
<script src="resource://zotero-unit/sinon-as-promised.js"></script>
<script src="resource://zotero-unit/pako_inflate.js"></script>
<script src="support.js" type="application/javascript;version=1.8"></script>
<script src="runtests.js" type="application/javascript;version=1.8"></script>
</body>

7
test/content/support.js
View file

@ -281,6 +281,13 @@ function clickOnItemsRow(itemsView, row, button = 0) {
}
/**
* Synchronous inflate
*/
function gunzip(gzdata) {
return pako.inflate(gzdata, { to: 'string' });
}
/**
* Get a default group used by all tests that want one, creating one if necessary

3075
test/resource/pako_inflate.js Normal file
View file

File diff suppressed because it is too large Load diff

5
test/resource/sinon.js
View file

@ -5278,6 +5278,11 @@ if (typeof sinon === "undefined") {
this.requestHeaders["Content-Type"] = "text/plain;charset=utf-8";
}
// Added by Zotero
if (this.requestHeaders['Content-Encoding'] == 'gzip') {
data = gunzip(data, true);
}
this.requestBody = data;
}

12
test/tests/utilities_internalTest.js
View file

@ -12,6 +12,18 @@ describe("Zotero.Utilities.Internal", function () {
})
describe("#gzip()/gunzip()", function () {
it("should compress and decompress a Unicode text string", function* () {
var text = "Voilà! \u1F429";
var compstr = yield Zotero.Utilities.Internal.gzip(text);
assert.isAbove(compstr.length, 0);
assert.notEqual(compstr.length, text.length);
var str = yield Zotero.Utilities.Internal.gunzip(compstr);
assert.equal(str, text);
});
});
describe("#delayGenerator", function () {
var spy;