From 827e7b51b5a76cbf7e44efe0725068de80224d73 Mon Sep 17 00:00:00 2001 From: John Crepezzi Date: Fri, 16 Feb 2018 09:52:44 -0500 Subject: [PATCH 1/8] Rewrite the memcached client * Update syntax to ES6 * Use `memcached` instead of `memcache` * Fix restrictions where expirations weren't pushed forward on GET * Fix a bug where we were unnecessarily bumping expirations on key search Closes #201 --- README.md | 4 +- config.js | 7 ++- lib/document_handler.js | 2 +- lib/document_stores/memcached.js | 83 +++++++++++++++++--------------- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index f01c68b..0383ff7 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,9 @@ All of which are optional except `type` with very logical default values. ### Memcached -To use memcached storage you must install the `memcache` package via npm +To use memcache storage you must install the `memcached` package via npm -`npm install memcache` +`npm install memcached` Once you've done that, your config section should look like: diff --git a/config.js b/config.js index 52ad004..4b33e28 100644 --- a/config.js +++ b/config.js @@ -33,10 +33,9 @@ }, "storage": { - "type": "redis", - "host": "0.0.0.0", - "port": 6379, - "db": 2, + "type": "memcached", + "host": "127.0.0.1", + "port": 11211, "expire": 2592000 }, diff --git a/lib/document_handler.js b/lib/document_handler.js index 68985ec..e5a6b56 100644 --- a/lib/document_handler.js +++ b/lib/document_handler.js @@ -123,7 +123,7 @@ DocumentHandler.prototype.chooseKey = function(callback) { } else { callback(key); } - }); + }, true); // Don't bump expirations when key searching }; DocumentHandler.prototype.acceptableKey = function() { diff --git a/lib/document_stores/memcached.js b/lib/document_stores/memcached.js index 2771886..be10db6 100644 --- a/lib/document_stores/memcached.js +++ b/lib/document_stores/memcached.js @@ -1,45 +1,52 @@ -var memcached = require('memcache'); -var winston = require('winston'); +const memcached = require('memcached'); +const winston = require('winston'); -// Create a new store with options -var MemcachedDocumentStore = function(options) { - this.expire = options.expire; - if (!MemcachedDocumentStore.client) { - MemcachedDocumentStore.connect(options); +class MemcachedDocumentStore { + + // Create a new store with options + constructor(options) { + this.expire = options.expire; + + const host = options.host || '127.0.0.1'; + const port = options.port || 11211; + const url = `${host}:${port}`; + this.connect(url); } -}; -// Create a connection -MemcachedDocumentStore.connect = function(options) { - var host = options.host || '127.0.0.1'; - var port = options.port || 11211; - this.client = new memcached.Client(port, host); - this.client.connect(); - this.client.on('connect', function() { - winston.info('connected to memcached on ' + host + ':' + port); - }); - this.client.on('error', function(e) { - winston.info('error connecting to memcached', { error: e }); - }); -}; + // Create a connection + connect(url) { + this.client = new memcached(url); -// Save file in a key -MemcachedDocumentStore.prototype.set = -function(key, data, callback, skipExpire) { - MemcachedDocumentStore.client.set(key, data, function(err) { - err ? callback(false) : callback(true); - }, skipExpire ? 0 : this.expire); -}; + winston.info(`connecting to memcached on ${url}`); -// Get a file from a key -MemcachedDocumentStore.prototype.get = function(key, callback, skipExpire) { - var _this = this; - MemcachedDocumentStore.client.get(key, function(err, reply) { - callback(err ? false : reply); - if (_this.expire && !skipExpire) { - winston.warn('store does not currently push forward expirations on GET'); - } - }); -}; + this.client.on('failure', function(error) { + winston.info('error connecting to memcached', {error}); + }); + } + + // Save file in a key + set(key, data, callback, skipExpire) { + this.client.set(key, data, skipExpire ? 0 : this.expire, (error) => { + callback(!error); + }); + } + + // Get a file from a key + get(key, callback, skipExpire) { + this.client.get(key, (error, data) => { + callback(error ? false : data); + + // Update the key so that the expiration is pushed forward + if (!skipExpire) { + this.set(key, data, (updateSucceeded) => { + if (!updateSucceeded) { + winston.error('failed to update expiration on GET', {key}); + } + }, skipExpire); + } + }); + } + +} module.exports = MemcachedDocumentStore; From 2b81e67ce7dbe63eedad447c4ab906a0cf00694f Mon Sep 17 00:00:00 2001 From: Razzeee Date: Mon, 9 Apr 2018 19:05:21 +0200 Subject: [PATCH 2/8] Update docs to real defaults --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f01c68b..2310f33 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ STDOUT. Check the README there for more details and usages. * `host` - the host the server runs on (default localhost) * `port` - the port the server runs on (default 7777) * `keyLength` - the length of the keys to user (default 10) -* `maxLength` - maximum length of a paste (default none) +* `maxLength` - maximum length of a paste (default 400000) * `staticMaxAge` - max age for static assets (86400) * `recompressStaticAssets` - whether or not to compile static js assets (true) * `documents` - static documents to serve (ex: http://hastebin.com/about.com) From dc0f151a7f0de6fc5480077e096e5ed87e0e573d Mon Sep 17 00:00:00 2001 From: Jacob Gunther Date: Sun, 15 Apr 2018 23:16:08 -0500 Subject: [PATCH 3/8] Fixed bug in RethinkDB document store and use classes --- lib/document_stores/rethinkdb.js | 70 +++++++++++++++++--------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/lib/document_stores/rethinkdb.js b/lib/document_stores/rethinkdb.js index b7ee7ae..431c2ab 100644 --- a/lib/document_stores/rethinkdb.js +++ b/lib/document_stores/rethinkdb.js @@ -1,39 +1,45 @@ const crypto = require('crypto'); const rethink = require('rethinkdbdash'); +const winston = require('winston'); -var RethinkDBStore = (options) => { - this.client = rethink({ - silent: true, - host: options.host || '127.0.0.1', - port: options.port || 28015, - db: options.db || 'haste', - user: options.user || 'admin', - password: options.password || '' - }); -}; +class RethinkDBStore { + constructor(options) { + this.client = rethink({ + silent: true, + host: options.host || '127.0.0.1', + port: options.port || 28015, + db: options.db || 'haste', + user: options.user || 'admin', + password: options.password || '' + }); + } -RethinkDBStore.md5 = (str) => { + set(key, data, callback) { + this.client.table('pastes').insert({ id: RethinkDBStore.md5(key), data: data }).run((error) => { + if (error) { + callback(false); + winston.error('failed to insert to table', error); + return; + } + callback(true); + }); + } + + get(key, callback) { + this.client.table('pastes').get(RethinkDBStore.md5(key)).run((error, result) => { + if (error || !result) { + callback(false); + winston.error('failed to insert to table', error); + return; + } + callback(result.data); + }); + } +} + +module.exports = RethinkDBStore; +module.exports.md5 = (str) => { const md5sum = crypto.createHash('md5'); md5sum.update(str); return md5sum.digest('hex'); -}; - -RethinkDBStore.prototype.set = (key, data, callback) => { - try { - this.client.table('uploads').insert({ id: RethinkDBStore.md5(key), data: data }).run((error) => { - if (error) return callback(false); - callback(true); - }); - } catch (err) { - callback(false); - } -}; - -RethinkDBStore.prototype.get = (key, callback) => { - this.client.table('uploads').get(RethinkDBStore.md5(key)).run((error, result) => { - if (error || !result) return callback(false); - callback(result.data); - }); -}; - -module.exports = RethinkDBStore; +}; \ No newline at end of file From 830dc1bc43b92ec3146a60e3b5c5d6da3540ee1f Mon Sep 17 00:00:00 2001 From: Jacob Gunther Date: Sun, 15 Apr 2018 23:16:39 -0500 Subject: [PATCH 4/8] Use uploads table --- lib/document_stores/rethinkdb.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/document_stores/rethinkdb.js b/lib/document_stores/rethinkdb.js index 431c2ab..04c5b6f 100644 --- a/lib/document_stores/rethinkdb.js +++ b/lib/document_stores/rethinkdb.js @@ -15,7 +15,7 @@ class RethinkDBStore { } set(key, data, callback) { - this.client.table('pastes').insert({ id: RethinkDBStore.md5(key), data: data }).run((error) => { + this.client.table('uploads').insert({ id: RethinkDBStore.md5(key), data: data }).run((error) => { if (error) { callback(false); winston.error('failed to insert to table', error); @@ -26,7 +26,7 @@ class RethinkDBStore { } get(key, callback) { - this.client.table('pastes').get(RethinkDBStore.md5(key)).run((error, result) => { + this.client.table('uploads').get(RethinkDBStore.md5(key)).run((error, result) => { if (error || !result) { callback(false); winston.error('failed to insert to table', error); From cd3bf26dbe8ed58f87af1b72a56320cbbd8efc36 Mon Sep 17 00:00:00 2001 From: Jacob Gunther <16949253+PassTheMayo@users.noreply.github.com> Date: Mon, 16 Apr 2018 10:52:53 -0500 Subject: [PATCH 5/8] Use local method for md5 --- lib/document_stores/rethinkdb.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/document_stores/rethinkdb.js b/lib/document_stores/rethinkdb.js index 04c5b6f..f945fa1 100644 --- a/lib/document_stores/rethinkdb.js +++ b/lib/document_stores/rethinkdb.js @@ -2,6 +2,12 @@ const crypto = require('crypto'); const rethink = require('rethinkdbdash'); const winston = require('winston'); +const md5 = (str) => { + const md5sum = crypto.createHash('md5'); + md5sum.update(str); + return md5sum.digest('hex'); +}; + class RethinkDBStore { constructor(options) { this.client = rethink({ @@ -15,7 +21,7 @@ class RethinkDBStore { } set(key, data, callback) { - this.client.table('uploads').insert({ id: RethinkDBStore.md5(key), data: data }).run((error) => { + this.client.table('uploads').insert({ id: md5(key), data: data }).run((error) => { if (error) { callback(false); winston.error('failed to insert to table', error); @@ -26,7 +32,7 @@ class RethinkDBStore { } get(key, callback) { - this.client.table('uploads').get(RethinkDBStore.md5(key)).run((error, result) => { + this.client.table('uploads').get(md5(key)).run((error, result) => { if (error || !result) { callback(false); winston.error('failed to insert to table', error); @@ -38,8 +44,3 @@ class RethinkDBStore { } module.exports = RethinkDBStore; -module.exports.md5 = (str) => { - const md5sum = crypto.createHash('md5'); - md5sum.update(str); - return md5sum.digest('hex'); -}; \ No newline at end of file From 5f6fefa7a6e702d82b8cc36df7ea183400bc0501 Mon Sep 17 00:00:00 2001 From: Jacob Gunther <16949253+PassTheMayo@users.noreply.github.com> Date: Mon, 30 Apr 2018 16:40:28 -0500 Subject: [PATCH 6/8] Fixed unnecessary logging when document not found --- lib/document_stores/rethinkdb.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/document_stores/rethinkdb.js b/lib/document_stores/rethinkdb.js index f945fa1..ca825af 100644 --- a/lib/document_stores/rethinkdb.js +++ b/lib/document_stores/rethinkdb.js @@ -35,7 +35,7 @@ class RethinkDBStore { this.client.table('uploads').get(md5(key)).run((error, result) => { if (error || !result) { callback(false); - winston.error('failed to insert to table', error); + if (error) winston.error('failed to insert to table', error); return; } callback(result.data); From b087ac8dd1c752f8036f7685ab24ec0b9375aa60 Mon Sep 17 00:00:00 2001 From: John Crepezzi Date: Thu, 12 Jul 2018 14:25:27 -0400 Subject: [PATCH 7/8] Added charset to raw content type Closes #230 --- lib/document_handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/document_handler.js b/lib/document_handler.js index e5a6b56..83eb141 100644 --- a/lib/document_handler.js +++ b/lib/document_handler.js @@ -36,7 +36,7 @@ DocumentHandler.prototype.handleRawGet = function(key, response, skipExpire) { this.store.get(key, function(ret) { if (ret) { winston.verbose('retrieved raw document', { key: key }); - response.writeHead(200, { 'content-type': 'text/plain' }); + response.writeHead(200, { 'content-type': 'text/plain; charset=UTF-8' }); response.end(ret); } else { From bf2b1c957a8ca2f8191ce6ba24b67e63158c361e Mon Sep 17 00:00:00 2001 From: John Crepezzi Date: Wed, 19 Sep 2018 10:37:34 -0400 Subject: [PATCH 8/8] Handle redis error and re-establish connection --- lib/document_stores/redis.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/document_stores/redis.js b/lib/document_stores/redis.js index 7edbaf0..eed07e7 100644 --- a/lib/document_stores/redis.js +++ b/lib/document_stores/redis.js @@ -29,6 +29,11 @@ RedisDocumentStore.connect = function(options) { if (options.password) { RedisDocumentStore.client.auth(options.password); } + + RedisDocumentStore.client.on('error', function(err) { + winston.error('redis disconnected', err); + }); + RedisDocumentStore.client.select(index, function(err) { if (err) { winston.error(