From 869fb6573829cb5044f6af91dc7cbe947c7e95ea Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Aug 2018 22:59:05 +0100 Subject: [PATCH 01/11] added googledatastore handler --- lib/document_stores/googledatastore.js | 92 ++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 lib/document_stores/googledatastore.js diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/googledatastore.js new file mode 100644 index 0000000..331b047 --- /dev/null +++ b/lib/document_stores/googledatastore.js @@ -0,0 +1,92 @@ +/*global require,module,process*/ + +const datastore = require('@google-cloud/datastore'); +const winston = require('winston'); + +class GoogleDatastoreDocumentStore { + + // Create a new store with options + constructor(options) { + this.kind = "Haste"; + this.expire = options.expire; + this.datastore = new Datastore(); + } + + // Save file in a key + set(key, data, callback, skipExpire) { + var now = new Date(); + var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000; + + var taskKey = this.datastore.key([this.kind, key]) + var task = { + key: taskKey, + data: [ + { + name: 'value', + value: data, + excludeFromIndexes: true + }, + { + name: 'expiration', + value: expireTime + } + ] + }; + + this.datastore.insert(task).then(() => { + callback(true); + }) + .catch(err => { + callback(false); + }); + } + + // Get a file from a key + get(key, callback, skipExpire) { + var taskKey = this.datastore.key([this.kind, key]) + + this.datastore.get(taskKey).then((entity) => { + + if (skipExpire) { + callback(entity[0]["value"]); + } + else { + // check for expiry + if (entity[0]["expiration"] != null && entity[0]["expiration"] < new Date()) { + winston.info("document expired", {key: key}); + callback(false); + } + else { + + // update expiry + var now = new Date(); + var task = { + key: taskKey, + data: [ + { + name: 'value', + value: entity[0]["value"], + excludeFromIndexes: true + }, + { + name: 'expiration', + value: new Date(now.getTime() + this.expire * 1000; + } + ] + }; + this.datastore.update(task).then(() => { + }) + .catch(err => { + winston.error("failed to update expiration", {error: err}); + }); + callback(entity[0]["value"]); + } + } + }) + .catch(err => { + winston.error("Error retrieving value from Google Datastore", {error: err}); + callback(false); + }); +} + +module.exports = GoogleDatastoreDocumentStore; From e3d18efdc6ab0ab746709574295b90adfe64557c Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Aug 2018 23:01:37 +0100 Subject: [PATCH 02/11] added npm package --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 9453b1e..650a0d6 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ }, "main": "haste", "dependencies": { + "@google-cloud/datastore": "^1.4.2", "connect-ratelimit": "0.0.7", "connect-route": "0.1.5", "connect": "3.4.1", From b6814a1445c58428072fc45302a1b2250f1a4dda Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Aug 2018 23:15:02 +0100 Subject: [PATCH 03/11] bugfixes --- lib/document_stores/googledatastore.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/googledatastore.js index 331b047..22ffe5d 100644 --- a/lib/document_stores/googledatastore.js +++ b/lib/document_stores/googledatastore.js @@ -1,6 +1,6 @@ /*global require,module,process*/ -const datastore = require('@google-cloud/datastore'); +const Datastore = require('@google-cloud/datastore'); const winston = require('winston'); class GoogleDatastoreDocumentStore { @@ -15,7 +15,7 @@ class GoogleDatastoreDocumentStore { // Save file in a key set(key, data, callback, skipExpire) { var now = new Date(); - var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000; + var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000); var taskKey = this.datastore.key([this.kind, key]) var task = { @@ -70,7 +70,7 @@ class GoogleDatastoreDocumentStore { }, { name: 'expiration', - value: new Date(now.getTime() + this.expire * 1000; + value: new Date(now.getTime() + this.expire * 1000) } ] }; @@ -87,6 +87,7 @@ class GoogleDatastoreDocumentStore { winston.error("Error retrieving value from Google Datastore", {error: err}); callback(false); }); + } } module.exports = GoogleDatastoreDocumentStore; From 6ebd72a86cfb3c78dca7e5d96b10e620f73987e7 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Aug 2018 23:34:56 +0100 Subject: [PATCH 04/11] updated readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index ba2bfc2..894ac85 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,20 @@ Also, you must create an `uploads` table, which will store all the data for uplo You can optionally add the `user` and `password` properties to use a user system. +### Google Datastore + +To use the Google Datastore storage system, you must install the `@google-cloud/datastore` package via npm + +`npm install @google-cloud/datastore` + +Once you've done that, your config section should look like this: + +```{ + "type": "googledatastore" +}``` + +Authentication can be handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started) + ## Author John Crepezzi From 2213c3874a8e0875aeee98a08635a0d7bb4606e2 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Aug 2018 23:48:19 +0100 Subject: [PATCH 05/11] updated readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 894ac85..649c469 100644 --- a/README.md +++ b/README.md @@ -206,9 +206,11 @@ To use the Google Datastore storage system, you must install the `@google-cloud/ Once you've done that, your config section should look like this: -```{ +``` json +{ "type": "googledatastore" -}``` +} +``` Authentication can be handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started) From b0bbb72f35f18e604f7e95639384f20f6d7088f0 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Aug 2018 00:35:09 +0100 Subject: [PATCH 06/11] updated to use Date(null) --- lib/document_stores/googledatastore.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/googledatastore.js index 22ffe5d..97385f1 100644 --- a/lib/document_stores/googledatastore.js +++ b/lib/document_stores/googledatastore.js @@ -15,7 +15,7 @@ class GoogleDatastoreDocumentStore { // Save file in a key set(key, data, callback, skipExpire) { var now = new Date(); - var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000); + var expireTime = skipExpire ? new Date(null) : new Date(now.getTime() + this.expire * 1000); var taskKey = this.datastore.key([this.kind, key]) var task = { @@ -52,7 +52,7 @@ class GoogleDatastoreDocumentStore { } else { // check for expiry - if (entity[0]["expiration"] != null && entity[0]["expiration"] < new Date()) { + if (entity[0]["expiration"] != new Date(null) && entity[0]["expiration"] < new Date()) { winston.info("document expired", {key: key}); callback(false); } From ebc749c5e004bcfcb983613875f853dbaa18b493 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Aug 2018 00:37:21 +0100 Subject: [PATCH 07/11] updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 649c469..9d8ea7d 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ Once you've done that, your config section should look like this: } ``` -Authentication can be handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started) +Authentication is handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started), by providing authentication details to the GOOGLE_APPLICATION_CREDENTIALS environmental variable. ## Author From 1eeef4ede46c1151c6f0f64cd76a4b2268b700e4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Aug 2018 01:21:37 +0100 Subject: [PATCH 08/11] restored using null --- lib/document_stores/googledatastore.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/googledatastore.js index 97385f1..6e6dcd3 100644 --- a/lib/document_stores/googledatastore.js +++ b/lib/document_stores/googledatastore.js @@ -15,7 +15,7 @@ class GoogleDatastoreDocumentStore { // Save file in a key set(key, data, callback, skipExpire) { var now = new Date(); - var expireTime = skipExpire ? new Date(null) : new Date(now.getTime() + this.expire * 1000); + var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000); var taskKey = this.datastore.key([this.kind, key]) var task = { @@ -47,13 +47,13 @@ class GoogleDatastoreDocumentStore { this.datastore.get(taskKey).then((entity) => { - if (skipExpire) { + if (skipExpire || entity[0]["expiration"] == null) { callback(entity[0]["value"]); } else { // check for expiry - if (entity[0]["expiration"] != new Date(null) && entity[0]["expiration"] < new Date()) { - winston.info("document expired", {key: key}); + if (entity[0]["expiration"] < new Date()) { + winston.info("document expired", {key: key, expiration: entity[0]["expiration"], check: new Date(null)}); callback(false); } else { From 5fb43eb67cd9f5e43ab217b83554375c8f50db55 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Aug 2018 01:28:26 +0100 Subject: [PATCH 09/11] added condition for this.expire not defined --- lib/document_stores/googledatastore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/googledatastore.js index 6e6dcd3..d951aec 100644 --- a/lib/document_stores/googledatastore.js +++ b/lib/document_stores/googledatastore.js @@ -15,7 +15,7 @@ class GoogleDatastoreDocumentStore { // Save file in a key set(key, data, callback, skipExpire) { var now = new Date(); - var expireTime = skipExpire ? null : new Date(now.getTime() + this.expire * 1000); + var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(now.getTime() + this.expire * 1000); var taskKey = this.datastore.key([this.kind, key]) var task = { From d41d7491d42f66fd71819a86e6d73eee71e01203 Mon Sep 17 00:00:00 2001 From: Yuan Gao <930832+meseta@users.noreply.github.com> Date: Sat, 1 Sep 2018 21:11:58 +0100 Subject: [PATCH 10/11] rename to google-datastore, and use Date.now() --- .../{googledatastore.js => google-datastore.js} | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) rename lib/document_stores/{googledatastore.js => google-datastore.js} (92%) diff --git a/lib/document_stores/googledatastore.js b/lib/document_stores/google-datastore.js similarity index 92% rename from lib/document_stores/googledatastore.js rename to lib/document_stores/google-datastore.js index d951aec..780780a 100644 --- a/lib/document_stores/googledatastore.js +++ b/lib/document_stores/google-datastore.js @@ -14,8 +14,7 @@ class GoogleDatastoreDocumentStore { // Save file in a key set(key, data, callback, skipExpire) { - var now = new Date(); - var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(now.getTime() + this.expire * 1000); + var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(Date.now() + this.expire * 1000); var taskKey = this.datastore.key([this.kind, key]) var task = { @@ -59,7 +58,6 @@ class GoogleDatastoreDocumentStore { else { // update expiry - var now = new Date(); var task = { key: taskKey, data: [ @@ -70,7 +68,7 @@ class GoogleDatastoreDocumentStore { }, { name: 'expiration', - value: new Date(now.getTime() + this.expire * 1000) + value: new Date(Date.now() + this.expire * 1000) } ] }; From 86bbc1899d37d34ecaf0016e07895e1f4576ebbf Mon Sep 17 00:00:00 2001 From: Yuan Gao <930832+meseta@users.noreply.github.com> Date: Sat, 1 Sep 2018 21:12:30 +0100 Subject: [PATCH 11/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d8ea7d..d571038 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ Once you've done that, your config section should look like this: ``` json { - "type": "googledatastore" + "type": "google-datastore" } ```