From b4c666fbcf554b3c4d20bd80a907b43cf0aaaf22 Mon Sep 17 00:00:00 2001 From: Jacob Parker Date: Fri, 28 Jun 2019 19:09:40 +0100 Subject: [PATCH 01/22] Add an Amazon S3 document store --- README.md | 21 ++++++++++++ lib/document_stores/amazon-s3.js | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 lib/document_stores/amazon-s3.js diff --git a/README.md b/README.md index ba2bfc2..c40be18 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,27 @@ Also, you must create an `uploads` table, which will store all the data for uplo You can optionally add the `user` and `password` properties to use a user system. +### Amazon S3 + +To use [Amazon S3](https://aws.amazon.com/s3/) as a storage system, you must +install the `aws-sdk` package via npm: + +`npm install aws-sdk` + +Once you've done that, your config section should look like this: + +```json +{ + "type": "amazon-s3", + "bucket": "your-bucket-name", + "region": "us-east-1" +} +``` + +Authentication is handled automatically by the client. Check +[Amazon's documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html) +for more information. + ## Author John Crepezzi diff --git a/lib/document_stores/amazon-s3.js b/lib/document_stores/amazon-s3.js new file mode 100644 index 0000000..11dd85d --- /dev/null +++ b/lib/document_stores/amazon-s3.js @@ -0,0 +1,56 @@ +/*global require,module,process*/ + +var AWS = require('aws-sdk'); +var winston = require('winston'); + +var AmazonS3DocumentStore = function(options) { + this.expire = options.expire; + this.bucket = options.bucket; + this.client = new AWS.S3({region: options.region}); +}; + +AmazonS3DocumentStore.prototype.get = function(key, callback, skipExpire) { + var _this = this; + + var req = { + Bucket: _this.bucket, + Key: key + }; + + _this.client.getObject(req, function(err, data) { + if(err) { + callback(false); + } + else { + callback(data.Body.toString('utf-8')); + if (_this.expire && !skipExpire) { + winston.warn('amazon s3 store cannot set expirations on keys'); + } + } + }); +} + +AmazonS3DocumentStore.prototype.set = function(key, data, callback, skipExpire) { + var _this = this; + + var req = { + Bucket: _this.bucket, + Key: key, + Body: data, + ContentType: 'text/plain' + }; + + _this.client.putObject(req, function(err, data) { + if (err) { + callback(false); + } + else { + callback(true); + if (_this.expire && !skipExpire) { + winston.warn('amazon s3 store cannot set expirations on keys'); + } + } + }); +} + +module.exports = AmazonS3DocumentStore; From 1fff48568f9789775b8e720bf9e20c7af942c77f Mon Sep 17 00:00:00 2001 From: Jacob Parker Date: Mon, 8 Jul 2019 16:59:04 +0100 Subject: [PATCH 02/22] Document the IAM permissions --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c40be18..23ca077 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,24 @@ Once you've done that, your config section should look like this: Authentication is handled automatically by the client. Check [Amazon's documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html) -for more information. +for more information. You will need to grant your role these permissions to +your bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::your-bucket-name-goes-here/*" + } + ] +} +``` ## Author From f6084b4339511813f30a23847cfa9d485f0059bc Mon Sep 17 00:00:00 2001 From: epdn Date: Mon, 18 May 2020 09:34:57 +0100 Subject: [PATCH 03/22] remove 1px margin from textarea, fixes useless scrollbar --- static/application.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/static/application.css b/static/application.css index 28087e3..15a06e9 100644 --- a/static/application.css +++ b/static/application.css @@ -17,6 +17,8 @@ textarea { outline: none; resize: none; font-size: 13px; + margin-top: 0; + margin-bottom: 0; } /* the line numbers */ From ce03749c2f28f06cfd1cf50ed98b0c37bc95400f Mon Sep 17 00:00:00 2001 From: Reece Dunham Date: Wed, 12 Aug 2020 23:45:18 +0000 Subject: [PATCH 04/22] Update dependencies to reduce security risk Signed-off-by: Reece Dunham --- package-lock.json | 527 ++++++++++++++++++++++++++++++---------------- package.json | 17 +- 2 files changed, 353 insertions(+), 191 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14685ee..cd5d970 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,11 @@ "integrity": "sha1-D8GqoIig4+8Ovi2IMbqw3PiEUGE=" }, "async-cache": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async-cache/-/async-cache-1.0.0.tgz", - "integrity": "sha1-yH9tgMcrOU7g+QYe3rJNjEtiKto=", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/async-cache/-/async-cache-1.1.0.tgz", + "integrity": "sha1-SppaidBl7F2OUlS9nulrp2xTK1o=", "requires": { - "lru-cache": "2.3.1" + "lru-cache": "^4.0.0" } }, "balanced-match": { @@ -23,30 +23,42 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "bl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", - "integrity": "sha1-/FQhoo/UImA2w7OJGmaiW8ZNIm4=", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", + "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", "requires": { - "readable-stream": "2.0.6" + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" }, "dependencies": { - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "readable-stream": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", - "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "1.0.7", - "string_decoder": "0.10.31", - "util-deprecate": "1.0.2" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" } } } @@ -57,7 +69,7 @@ "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", "dev": true, "requires": { - "balanced-match": "1.0.0", + "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, @@ -67,10 +79,19 @@ "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", "dev": true }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "buffer-writer": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz", - "integrity": "sha1-bCnDst6gyeRVofJhoZmkigT4iwg=" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz", + "integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg=" }, "busboy": { "version": "0.2.4", @@ -78,7 +99,7 @@ "integrity": "sha1-GXfpbh7ohGSWUevfVIypAHWLp/M=", "requires": { "dicer": "0.2.3", - "readable-stream": "1.1.14" + "readable-stream": "1.1.x" } }, "colors": { @@ -86,15 +107,6 @@ "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz", "integrity": "sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=" }, - "commander": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", - "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=", - "dev": true, - "requires": { - "graceful-readlink": "1.0.1" - } - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -102,14 +114,29 @@ "dev": true }, "connect": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/connect/-/connect-3.4.1.tgz", - "integrity": "sha1-ohNh0/QJnvdhzabcSpc7seuwo00=", + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/connect/-/connect-3.6.6.tgz", + "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=", "requires": { - "debug": "2.2.0", - "finalhandler": "0.4.1", - "parseurl": "1.3.1", - "utils-merge": "1.0.0" + "debug": "2.6.9", + "finalhandler": "1.1.0", + "parseurl": "~1.3.2", + "utils-merge": "1.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, "connect-ratelimit": { @@ -122,6 +149,11 @@ "resolved": "https://registry.npmjs.org/connect-route/-/connect-route-0.1.5.tgz", "integrity": "sha1-48IYMZ0uiKiprgsOD+Cacpw5dEo=" }, + "connection-parse": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", + "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=" + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -132,34 +164,25 @@ "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" }, - "debug": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", - "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", - "requires": { - "ms": "0.7.1" - } - }, "dicer": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.3.tgz", "integrity": "sha1-8AKBGJpVwjUe+ASQpP6fssWcSTk=", "requires": { - "readable-stream": "1.1.14", + "readable-stream": "1.1.x", "streamsearch": "0.1.2" } }, - "diff": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.2.0.tgz", - "integrity": "sha1-yc45Okt8vQsFinJck98pkCeGj/k=", - "dev": true - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -177,19 +200,37 @@ "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" }, "fd": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/fd/-/fd-0.0.2.tgz", - "integrity": "sha1-4O2yvXqIzIbdnxY5HLqDJBj9h+4=" + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/fd/-/fd-0.0.3.tgz", + "integrity": "sha512-iAHrIslQb3U68OcMSP0kkNWabp7sSN6d2TBSb2JO3gcLJVDd4owr/hKM4SFJovFOUeeXeItjYgouEDTMWiVAnA==" }, "finalhandler": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz", - "integrity": "sha1-haF8bFmpRxfSYtYSMNSw6+PUoU0=", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz", + "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", "requires": { - "debug": "2.2.0", - "escape-html": "1.0.3", - "on-finished": "2.3.0", - "unpipe": "1.0.0" + "debug": "2.6.9", + "encodeurl": "~1.0.1", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.3.1", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, "fs.realpath": { @@ -199,47 +240,24 @@ "dev": true }, "generic-pool": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz", - "integrity": "sha1-rwTcLDJc/Ll1Aj+lK/zpYXp0Nf0=" - }, - "glob": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", - "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=", - "dev": true, - "requires": { - "fs.realpath": "1.0.0", - "inflight": "1.0.6", - "inherits": "2.0.3", - "minimatch": "3.0.4", - "once": "1.4.0", - "path-is-absolute": "1.0.1" - } + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.2.tgz", + "integrity": "sha1-iGvFvwvrfblugby7oHiBjeWmJoM=" }, "graceful-fs": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", "optional": true }, - "graceful-readlink": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", - "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=", - "dev": true - }, - "growl": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.9.2.tgz", - "integrity": "sha1-Dqd0NxXbjY3ixe3hd14bRayFwC8=", - "dev": true - }, - "has-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", - "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", - "dev": true + "hashring": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", + "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", + "requires": { + "connection-parse": "0.0.x", + "simple-lru-cache": "0.0.x" + } }, "he": { "version": "1.1.1", @@ -247,14 +265,19 @@ "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", "dev": true }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "requires": { - "once": "1.4.0", - "wrappy": "1.0.2" + "once": "^1.3.0", + "wrappy": "1" } }, "inherits": { @@ -267,15 +290,41 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" }, + "jackpot": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/jackpot/-/jackpot-0.0.6.tgz", + "integrity": "sha1-PP8GQoXL9m9OqyWTyQvOgWqCGEk=", + "requires": { + "retry": "0.6.0" + } + }, + "js-string-escape": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", + "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8=" + }, "lru-cache": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.3.1.tgz", - "integrity": "sha1-s632s9hW6VTiw5DmzvIggSRaU9Y=" + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "memcached": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/memcached/-/memcached-2.2.2.tgz", + "integrity": "sha1-aPhsz9hLz5PMJe1G1tf8DHUhydU=", + "requires": { + "hashring": "3.2.x", + "jackpot": ">=0.0.6" + } }, "mime": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.6.tgz", - "integrity": "sha1-WR2E02U6awtKO5343lqoEI5y5eA=" + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "minimatch": { "version": "3.0.4", @@ -283,7 +332,7 @@ "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, "requires": { - "brace-expansion": "1.1.8" + "brace-expansion": "^1.1.7" } }, "minimist": { @@ -308,26 +357,85 @@ "dev": true, "requires": { "browser-stdout": "1.3.0", - "commander": "2.9.0", - "debug": "2.2.0", - "diff": "3.2.0", + "commander": "2.11.0", + "debug": "3.1.0", + "diff": "3.3.1", "escape-string-regexp": "1.0.5", - "glob": "7.1.1", - "growl": "1.9.2", + "glob": "7.1.2", + "growl": "1.10.3", "he": "1.1.1", "mkdirp": "0.5.1", - "supports-color": "3.1.2" + "supports-color": "4.4.0" + }, + "dependencies": { + "commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "diff": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", + "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", + "dev": true + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "growl": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", + "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", + "dev": true + }, + "has-flag": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", + "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", + "dev": true + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "supports-color": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "dev": true, + "requires": { + "has-flag": "^2.0.0" + } + } } }, - "ms": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", - "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" - }, "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, "on-finished": { "version": "2.3.0", @@ -343,7 +451,7 @@ "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "requires": { - "wrappy": "1.0.2" + "wrappy": "1" } }, "packet-reader": { @@ -352,9 +460,9 @@ "integrity": "sha1-gZ300BC4LV6lZx+KGjrPA5vNdwA=" }, "parseurl": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz", - "integrity": "sha1-yKuMkiO6NIiKpkopeyiFO+wY2lY=" + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-is-absolute": { "version": "1.0.1", @@ -363,17 +471,18 @@ "dev": true }, "pg": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-4.1.1.tgz", - "integrity": "sha1-mEgKz8089qP5Yhyl1FiUFVgqVzI=", + "version": "4.5.7", + "resolved": "https://registry.npmjs.org/pg/-/pg-4.5.7.tgz", + "integrity": "sha1-Ra4WsjcGpjRaAyed7MaveVwW0ps=", "requires": { - "buffer-writer": "1.0.0", - "generic-pool": "2.1.1", + "buffer-writer": "1.0.1", + "generic-pool": "2.4.2", + "js-string-escape": "1.0.1", "packet-reader": "0.2.0", "pg-connection-string": "0.1.3", - "pg-types": "1.6.0", + "pg-types": "1.*", "pgpass": "0.0.3", - "semver": "4.3.6" + "semver": "^4.1.0" } }, "pg-connection-string": { @@ -381,17 +490,29 @@ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, "pg-types": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.6.0.tgz", - "integrity": "sha1-OHKg8ZkUMCVJf07ipl/a8A1+qLM=" + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz", + "integrity": "sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~1.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.0", + "postgres-interval": "^1.1.0" + } }, "pgpass": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-0.0.3.tgz", "integrity": "sha1-EuZ+NDsxicLzEgbrycwL7//PkUA=", "requires": { - "split": "0.3.3" + "split": "~0.3" } }, "pkginfo": { @@ -399,20 +520,43 @@ "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.2.3.tgz", "integrity": "sha1-cjnEKl72wwuPMoQ52bn/cQQkkPg=" }, - "process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + "postgres-array": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz", + "integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.6.tgz", + "integrity": "sha512-o2a4gxeFcox+CgB3Ig/kNHBP23PiEXHCXx7pcIIsvzoNz4qv+lKTyiSkjOXIMNUl12MO/mOYl2K6wR9X5K6Plg==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "readable-stream": { "version": "1.1.14", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", "isarray": "0.0.1", - "string_decoder": "0.10.31" + "string_decoder": "~0.10.x" } }, "redis": { @@ -425,7 +569,7 @@ "resolved": "https://registry.npmjs.org/redis-url/-/redis-url-0.1.0.tgz", "integrity": "sha1-TaXlsYG2wMrW4aVcf1Co5u53ebs=", "requires": { - "redis": "0.8.1" + "redis": ">= 0.0.1" } }, "request": { @@ -433,11 +577,26 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.9.203.tgz", "integrity": "sha1-bBcRpUB/uUoRQhlWPkQUW8v0cjo=" }, + "retry": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.6.0.tgz", + "integrity": "sha1-HAEHEyeab9Ho3vKK8MP/GHHKpTc=" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "semver": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=" }, + "simple-lru-cache": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", + "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=" + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -448,20 +607,20 @@ "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", "requires": { - "through": "2.3.8" + "through": "2" } }, "st": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/st/-/st-1.1.0.tgz", - "integrity": "sha1-c7ltsLdkTZp4zjg0o+T37G6Hz3Y=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/st/-/st-2.0.0.tgz", + "integrity": "sha512-drN+aGYnrZPNYIymmNwIY7LXYJ8MqsqXj4fMRue3FOgGMdGjSX10fhJ3qx0sVQPhcWxhEaN4U/eWM4O4dbYNAw==", "requires": { - "async-cache": "1.0.0", - "bl": "1.0.3", - "fd": "0.0.2", - "graceful-fs": "4.1.11", - "mime": "1.3.6", - "negotiator": "0.6.1" + "async-cache": "^1.1.0", + "bl": "^4.0.0", + "fd": "~0.0.2", + "graceful-fs": "^4.2.3", + "mime": "^2.4.4", + "negotiator": "~0.6.2" } }, "stack-trace": { @@ -469,6 +628,11 @@ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, + "statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" + }, "streamsearch": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", @@ -479,15 +643,6 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" }, - "supports-color": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz", - "integrity": "sha1-cqJiiU2dQIuVbKBf83su2KbiotU=", - "dev": true, - "requires": { - "has-flag": "1.0.0" - } - }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -498,8 +653,8 @@ "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.1.6.tgz", "integrity": "sha512-/rseyxEKEVMBo8279lqpoJgD6C/i/CIi+9TJDvWmb+Xo6mqMKwjA8Io3IMHlcXQzj99feR6zrN8m3wqqvm/nYA==", "requires": { - "commander": "2.11.0", - "source-map": "0.6.1" + "commander": "~2.11.0", + "source-map": "~0.6.1" }, "dependencies": { "commander": { @@ -520,22 +675,22 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz", - "integrity": "sha1-ApT7kiu5N1FTVBxPcJYjHyh8ivg=" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "winston": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/winston/-/winston-0.6.2.tgz", "integrity": "sha1-QUT+JYbNwZphK/jANVkBMskGS9I=", "requires": { - "async": "0.1.22", - "colors": "0.6.2", - "cycle": "1.0.3", - "eyes": "0.1.8", - "pkginfo": "0.2.3", - "request": "2.9.203", - "stack-trace": "0.0.10" + "async": "0.1.x", + "colors": "0.x.x", + "cycle": "1.0.x", + "eyes": "0.1.x", + "pkginfo": "0.2.x", + "request": "2.9.x", + "stack-trace": "0.0.x" } }, "wrappy": { @@ -543,6 +698,16 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" } } } diff --git a/package.json b/package.json index 9453b1e..239c75a 100644 --- a/package.json +++ b/package.json @@ -14,25 +14,22 @@ }, "main": "haste", "dependencies": { + "busboy": "0.2.4", + "connect": "3.6.6", "connect-ratelimit": "0.0.7", "connect-route": "0.1.5", - "connect": "3.4.1", - "st": "1.1.0", - "winston": "0.6.2", - "redis-url": "0.1.0", + "memcached": "^2.2.2", + "pg": "4.5.7", "redis": "0.8.1", + "redis-url": "0.1.0", + "st": "2.0.0", "uglify-js": "3.1.6", - "busboy": "0.2.4", - "pg": "4.1.1" + "winston": "0.6.2" }, "devDependencies": { "mocha": "^4.0.1" }, "bundledDependencies": [], - "engines": { - "node": "8.1.4", - "npm": "5.2.0" - }, "bin": { "haste-server": "./server.js" }, From 655f2af45ad35b9f44b74e8c86a7aaf248983f74 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 17:33:01 +0200 Subject: [PATCH 05/22] script for turning env-vars into config.js --- docker-entrypoint.sh | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..d9a99a1 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# We use this file to translate environmental variables to .env files used by the application + +set -e +function create_document_string () { + IFS=',' read -ra doc_array <<< "$DOCUMENTS" + document_string="{" + + for i in "${doc_array[@]}" + do + IFS='=' read -ra document <<< "$i" + document_string+="\"${document[0]}\": \"${document[1]}\"," + done + + # Remove trailing "," + [[ "$document_string" == *, ]] && document_string=${document_string::${#document_string}-1} + document_string+="}" + echo $document_string +} + +document_string=$(create_document_string) + +echo " +{ + + \"host\": \"$HOST\", + \"port\": ${PORT}, + + \"keyLength\": $KEY_LENGTH, + + \"maxLength\": $MAX_LENGTH, + + \"staticMaxAge\": $STATIC_MAX_AGE, + + \"recompressStaticAssets\": $RECOMPRESS_STATIC_ASSETS, + + \"logging\": [ + { + \"level\": \"$LOGGING_LEVEL\", + \"type\": \"$LOGGING_TYPE\", + \"colorize\": $LOGGING_COLORIZE + } + ], + + \"keyGenerator\": { + \"type\": \"$KEYGENERATOR_TYPE\" + }, + + \"rateLimits\": { + + \"categories\": { + \"normal\": { + \"totalRequests\": $RATELIMITS_NORMAL_TOTAL_REQUESTS, + \"every\": $RATELIMITS_NORMAL_EVERY_SECONDS + }, + \"whitelist\": { + \"totalRequests\": $RATELIMITS_WHITELIST_TOTAL_REQUESTS, + \"every\": $RATELIMITS_WHITELIST_EVERY_SECONDS + }, + \"blacklist\": { + \"totalRequests\": $RATELIMITS_BLACKLIST_TOTAL_REQUESTS, + \"every\": $RATELIMITS_BLACKLIST_EVERY_SECONDS + } + } + }, + + \"storage\": { + \"type\": \"$STORAGE_TYPE\", + \"host\": \"$STORAGE_HOST\", + \"port\": $STORAGE_PORT, + \"expire\": $STORAGE_EXPIRE_SECONDS, + \"db\": $STORAGE_DB + }, + + \"documents\": $document_string + +} +" > config.js + + + +exec "$@" \ No newline at end of file From 064680003d2634a5e9a06750336dd93c4941d445 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 17:33:19 +0200 Subject: [PATCH 06/22] basic dockerfile with default env vars --- Dockerfile | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d3e42c1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +FROM node:latest + +RUN mkdir -p /usr/src/app && \ + chown node:node /usr/src/app + +USER node:node + +WORKDIR /usr/src/app + +COPY --chown=node:node . . + +RUN npm install && \ + npm install redis && \ + npm install pg && \ + npm install memcached + +ENV STORAGE_TYPE=memcached \ + STORAGE_HOST=127.0.0.1 \ + STORAGE_PORT=11211\ + STORAGE_EXPIRE_SECONDS=2592000\ + STORAGE_DB=2 \ + STORAGE_CONNECTION_URL= \ + STORAGE_BUCKET= \ + STORAGE_REGION= + +ENV LOGGING_LEVEL=verbose \ + LOGGING_TYPE=Console \ + LOGGING_COLORIZE=true + +ENV HOST=0.0.0.0\ + PORT=7777\ + KEY_LENGTH=10\ + MAX_LENGTH=400000\ + STATIC_MAX_AGE=86400\ + RECOMPRESS_STATIC_ASSETS=true + +ENV KEYGENERATOR_TYPE=phonetic \ + KEYGENERATOR_KEYSPACE= + +ENV RATELIMITS_NORMAL_TOTAL_REQUESTS=500\ + RATELIMITS_NORMAL_EVERY_SECONDS=60000 \ + RATELIMITS_WHITELIST_TOTAL_REQUESTS=500\ + RATELIMITS_WHITELIST_EVERY_SECONDS=60000 \ + # comma separated list for the whitelisted \ + RATELIMITS_WHITELIST= \ + \ + RATELIMITS_BLACKLIST_TOTAL_REQUESTS=60000 \ + RATELIMITS_BLACKLIST_EVERY_SECONDS=500 \ + # comma separated list for the blacklisted \ + RATELIMITS_BLACKLIST=example.com,example.org +ENV DOCUMENTS=about=./about.md + +EXPOSE 7777 + +ENTRYPOINT [ "bash", "docker-entrypoint.sh" ] + +CMD ["npm", "start"] From 0612ba001ec2ad8cc30a1c11fda58003b95ab4eb Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 17:33:40 +0200 Subject: [PATCH 07/22] basic docker-compose for running the project together with memcached --- docker-compose.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..8365c5d --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +version: '3.0' +services: + haste-server: + build: . + networks: + - db-network + environment: + - STORAGE_TYPE=memcached + - STORAGE_HOST=memcached + - STORAGE_PORT=11211 + ports: + - 7777:7777 + memcached: + image: memcached:latest + networks: + - db-network + +networks: + db-network: From edd428ff3734ebb0ce6d4d824d11b4ae7d9ce0e5 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:43:39 +0200 Subject: [PATCH 08/22] fix some names for env vars --- Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3e42c1..4f0a949 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,10 @@ ENV STORAGE_TYPE=memcached \ STORAGE_EXPIRE_SECONDS=2592000\ STORAGE_DB=2 \ STORAGE_CONNECTION_URL= \ - STORAGE_BUCKET= \ - STORAGE_REGION= + STORAGE_AWS_BUCKET= \ + STORAGE_AWS_REGION= \ + STORAGE_PG_PASSWORD= \ + STORAGE_PG_USERNAME= ENV LOGGING_LEVEL=verbose \ LOGGING_TYPE=Console \ @@ -39,15 +41,15 @@ ENV KEYGENERATOR_TYPE=phonetic \ ENV RATELIMITS_NORMAL_TOTAL_REQUESTS=500\ RATELIMITS_NORMAL_EVERY_SECONDS=60000 \ - RATELIMITS_WHITELIST_TOTAL_REQUESTS=500\ - RATELIMITS_WHITELIST_EVERY_SECONDS=60000 \ + RATELIMITS_WHITELIST_TOTAL_REQUESTS= \ + RATELIMITS_WHITELIST_EVERY_SECONDS= \ # comma separated list for the whitelisted \ - RATELIMITS_WHITELIST= \ + RATELIMITS_WHITELIST=example1.whitelist,example2.whitelist \ \ - RATELIMITS_BLACKLIST_TOTAL_REQUESTS=60000 \ - RATELIMITS_BLACKLIST_EVERY_SECONDS=500 \ + RATELIMITS_BLACKLIST_TOTAL_REQUESTS= \ + RATELIMITS_BLACKLIST_EVERY_SECONDS= \ # comma separated list for the blacklisted \ - RATELIMITS_BLACKLIST=example.com,example.org + RATELIMITS_BLACKLIST=example1.blacklist,example2.blacklist ENV DOCUMENTS=about=./about.md EXPOSE 7777 From 36c854ef1bdc3305c8f10c55688c137e6bb5e8c4 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:44:32 +0200 Subject: [PATCH 09/22] move creating the config file to a js file --- docker-entrypoint.js | 105 +++++++++++++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 76 +------------------------------ 2 files changed, 106 insertions(+), 75 deletions(-) create mode 100644 docker-entrypoint.js diff --git a/docker-entrypoint.js b/docker-entrypoint.js new file mode 100644 index 0000000..d902fb8 --- /dev/null +++ b/docker-entrypoint.js @@ -0,0 +1,105 @@ +const { promisify } = require("util"); +const { + HOST, + PORT, + KEY_LENGTH, + MAX_LENGTH, + STATIC_MAX_AGE, + RECOMPRESS_STATIC_ASSETS, + STORAGE_TYPE, + STORAGE_HOST, + STORAGE_PORT, + STORAGE_EXPIRE_SECONDS, + STORAGE_DB, + STORAGE_AWS_BUCKET, + STORAGE_AWS_REGION, + STORAGE_PG_PASSWORD, + STORAGE_PG_USERNAME, + LOGGING_LEVEL, + LOGGING_TYPE, + LOGGING_COLORIZE, + KEYGENERATOR_TYPE, + KEY_GENERATOR_KEYSPACE, + RATE_LIMITS_NORMAL_TOTAL_REQUESTS, + RATE_LIMITS_NORMAL_EVERY_SECONDS, + RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, + RATE_LIMITS_WHITELIST_EVERY_SECONDS, + RATE_LIMITS_WHITELIST, + RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS, + RATE_LIMITS_BLACKLIST_EVERY_SECONDS, + RATE_LIMITS_BLACKLIST, + DOCUMENTS, +} = process.env; + +const config = { + host: HOST, + port: PORT, + + keyLength: KEY_LENGTH, + + maxLength: MAX_LENGTH, + + staticMaxAge: STATIC_MAX_AGE, + + recompressStaticAssets: RECOMPRESS_STATIC_ASSETS, + + logging: [ + { + level: LOGGING_LEVEL, + type: LOGGING_TYPE, + colorize: LOGGING_COLORIZE, + }, + ], + + keyGenerator: { + type: KEYGENERATOR_TYPE, + keyspace: KEY_GENERATOR_KEYSPACE, + }, + + rateLimits: { + whitelist: RATE_LIMITS_WHITELIST ? RATE_LIMITS_WHITELIST.split(",") : [], + blacklist: RATE_LIMITS_BLACKLIST ? RATE_LIMITS_BLACKLIST.split(",") : [], + categories: { + normal: { + totalRequests: RATE_LIMITS_NORMAL_TOTAL_REQUESTS, + every: RATE_LIMITS_NORMAL_EVERY_SECONDS, + }, + whitelist: + RATE_LIMITS_WHITELIST_EVERY_SECONDS || + RATE_LIMITS_WHITELIST_TOTAL_REQUESTS + ? { + totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, + every: RATE_LIMITS_WHITELIST_EVERY_SECONDS, + } + : null, + blacklist: + RATE_LIMITS_BLACKLIST_EVERY_SECONDS || + RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS + ? { + totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, + every: RATE_LIMITS_BLACKLIST_EVERY_SECONDS, + } + : null, + }, + }, + + storage: { + type: STORAGE_TYPE, + host: STORAGE_HOST, + port: STORAGE_PORT, + expire: STORAGE_EXPIRE_SECONDS, + bucket: STORAGE_AWS_BUCKET, + region: STORAGE_AWS_REGION, + connectionUrl: `postgres://${STORAGE_PG_USERNAME}:${STORAGE_PG_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`, + db: STORAGE_DB, + }, + + documents: DOCUMENTS + ? DOCUMENTS.split(",").reduce((acc, item) => { + const keyAndValueArray = item.replace(/\s/g, "").split("="); + return { ...acc, [keyAndValueArray[0]]: keyAndValueArray[1] }; + }, {}) + : null, +}; + +console.log(JSON.stringify(config)); diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index d9a99a1..8e4e5a1 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -3,81 +3,7 @@ # We use this file to translate environmental variables to .env files used by the application set -e -function create_document_string () { - IFS=',' read -ra doc_array <<< "$DOCUMENTS" - document_string="{" - - for i in "${doc_array[@]}" - do - IFS='=' read -ra document <<< "$i" - document_string+="\"${document[0]}\": \"${document[1]}\"," - done - - # Remove trailing "," - [[ "$document_string" == *, ]] && document_string=${document_string::${#document_string}-1} - document_string+="}" - echo $document_string -} - -document_string=$(create_document_string) - -echo " -{ - - \"host\": \"$HOST\", - \"port\": ${PORT}, - - \"keyLength\": $KEY_LENGTH, - - \"maxLength\": $MAX_LENGTH, - - \"staticMaxAge\": $STATIC_MAX_AGE, - - \"recompressStaticAssets\": $RECOMPRESS_STATIC_ASSETS, - - \"logging\": [ - { - \"level\": \"$LOGGING_LEVEL\", - \"type\": \"$LOGGING_TYPE\", - \"colorize\": $LOGGING_COLORIZE - } - ], - - \"keyGenerator\": { - \"type\": \"$KEYGENERATOR_TYPE\" - }, - - \"rateLimits\": { - - \"categories\": { - \"normal\": { - \"totalRequests\": $RATELIMITS_NORMAL_TOTAL_REQUESTS, - \"every\": $RATELIMITS_NORMAL_EVERY_SECONDS - }, - \"whitelist\": { - \"totalRequests\": $RATELIMITS_WHITELIST_TOTAL_REQUESTS, - \"every\": $RATELIMITS_WHITELIST_EVERY_SECONDS - }, - \"blacklist\": { - \"totalRequests\": $RATELIMITS_BLACKLIST_TOTAL_REQUESTS, - \"every\": $RATELIMITS_BLACKLIST_EVERY_SECONDS - } - } - }, - - \"storage\": { - \"type\": \"$STORAGE_TYPE\", - \"host\": \"$STORAGE_HOST\", - \"port\": $STORAGE_PORT, - \"expire\": $STORAGE_EXPIRE_SECONDS, - \"db\": $STORAGE_DB - }, - - \"documents\": $document_string - -} -" > config.js - +node ./docker-entrypoint.js > ./config.js exec "$@" \ No newline at end of file From aef4bb5edbc98b245280ccd1db7c766ca294d438 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:45:27 +0200 Subject: [PATCH 10/22] add dockerignore file --- .dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a865156 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +npm-debug.log +node_modules +*.swp +*.swo +data +*.DS_Store From 0f145b4444f8c35c35aba8818c355c15c168ebf4 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:48:32 +0200 Subject: [PATCH 11/22] pin versions --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f0a949..136b5b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:latest +FROM node:14.8.0-stretch RUN mkdir -p /usr/src/app && \ chown node:node /usr/src/app @@ -10,9 +10,9 @@ WORKDIR /usr/src/app COPY --chown=node:node . . RUN npm install && \ - npm install redis && \ - npm install pg && \ - npm install memcached + npm install redis@0.8.1 && \ + npm install pg@4.1.1 && \ + npm install memcached@2.2.2 ENV STORAGE_TYPE=memcached \ STORAGE_HOST=127.0.0.1 \ From 5a8d52a5e3e76e735cf135e33596f70cc548b138 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:56:11 +0200 Subject: [PATCH 12/22] add healthcheck, and stopsignal, plus export the correct port --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 136b5b6..bcbfe43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,8 +52,10 @@ ENV RATELIMITS_NORMAL_TOTAL_REQUESTS=500\ RATELIMITS_BLACKLIST=example1.blacklist,example2.blacklist ENV DOCUMENTS=about=./about.md -EXPOSE 7777 - +EXPOSE ${PORT} +STOPSIGNAL SIGINT ENTRYPOINT [ "bash", "docker-entrypoint.sh" ] +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s \ + --retries=3 CMD [ "curl" , "-f" "localhost:${PORT}", "||", "exit", "1"] CMD ["npm", "start"] From 5c9311fb85292d7a563287b99ea5f190d855205a Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 20:56:45 +0200 Subject: [PATCH 13/22] remove unused import --- docker-entrypoint.js | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-entrypoint.js b/docker-entrypoint.js index d902fb8..e9e7ead 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -1,4 +1,3 @@ -const { promisify } = require("util"); const { HOST, PORT, From 9f419935668aa429f4bc1c7694ef4f6edfb9d2ee Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 21:06:02 +0200 Subject: [PATCH 14/22] also install rethinkdb and aws-sdk --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bcbfe43..76b87f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,9 @@ COPY --chown=node:node . . RUN npm install && \ npm install redis@0.8.1 && \ npm install pg@4.1.1 && \ - npm install memcached@2.2.2 + npm install memcached@2.2.2 && \ + npm install aws-sdk@2.738.0 && \ + npm install rethinkdbdash@2.3.31 ENV STORAGE_TYPE=memcached \ STORAGE_HOST=127.0.0.1 \ From 69cf505a9041a47f35d30ce4150e0c47f71473b7 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 21:29:41 +0200 Subject: [PATCH 15/22] remove pg connect string, add rethink user and password --- Dockerfile | 5 +++-- docker-entrypoint.js | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76b87f3..f4f2527 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,11 +21,12 @@ ENV STORAGE_TYPE=memcached \ STORAGE_PORT=11211\ STORAGE_EXPIRE_SECONDS=2592000\ STORAGE_DB=2 \ - STORAGE_CONNECTION_URL= \ STORAGE_AWS_BUCKET= \ STORAGE_AWS_REGION= \ STORAGE_PG_PASSWORD= \ - STORAGE_PG_USERNAME= + STORAGE_PG_USERNAME= \ + STORAGE_RETHINK_USER= \ + STORAGE_RETHINK_PASSWORD= ENV LOGGING_LEVEL=verbose \ LOGGING_TYPE=Console \ diff --git a/docker-entrypoint.js b/docker-entrypoint.js index e9e7ead..d661fbf 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -14,6 +14,8 @@ const { STORAGE_AWS_REGION, STORAGE_PG_PASSWORD, STORAGE_PG_USERNAME, + STORAGE_RETHINK_USER, + STORAGE_RETHINK_PASSWORD, LOGGING_LEVEL, LOGGING_TYPE, LOGGING_COLORIZE, @@ -91,6 +93,8 @@ const config = { region: STORAGE_AWS_REGION, connectionUrl: `postgres://${STORAGE_PG_USERNAME}:${STORAGE_PG_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`, db: STORAGE_DB, + user: STORAGE_RETHINK_USER, + password: STORAGE_RETHINK_USER, }, documents: DOCUMENTS From 05ecc90764dce2b6e24e8f8be3da707e8d77679f Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 21:47:48 +0200 Subject: [PATCH 16/22] add file path --- Dockerfile | 3 ++- docker-entrypoint.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f4f2527..3b1f0e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,8 @@ ENV STORAGE_TYPE=memcached \ STORAGE_PG_PASSWORD= \ STORAGE_PG_USERNAME= \ STORAGE_RETHINK_USER= \ - STORAGE_RETHINK_PASSWORD= + STORAGE_RETHINK_PASSWORD= \ + STORAGE_FILEPATH= ENV LOGGING_LEVEL=verbose \ LOGGING_TYPE=Console \ diff --git a/docker-entrypoint.js b/docker-entrypoint.js index d661fbf..60be55d 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -16,6 +16,7 @@ const { STORAGE_PG_USERNAME, STORAGE_RETHINK_USER, STORAGE_RETHINK_PASSWORD, + STORAGE_FILEPATH, LOGGING_LEVEL, LOGGING_TYPE, LOGGING_COLORIZE, @@ -95,6 +96,7 @@ const config = { db: STORAGE_DB, user: STORAGE_RETHINK_USER, password: STORAGE_RETHINK_USER, + path: STORAGE_FILEPATH, }, documents: DOCUMENTS From 342f56ce1a9513716249ec54d4b62e09401cb97c Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 21:56:58 +0200 Subject: [PATCH 17/22] use same password and username env vars for all types --- Dockerfile | 6 ++---- docker-entrypoint.js | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b1f0e2..34a06e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,10 +23,8 @@ ENV STORAGE_TYPE=memcached \ STORAGE_DB=2 \ STORAGE_AWS_BUCKET= \ STORAGE_AWS_REGION= \ - STORAGE_PG_PASSWORD= \ - STORAGE_PG_USERNAME= \ - STORAGE_RETHINK_USER= \ - STORAGE_RETHINK_PASSWORD= \ + STORAGE_USENAMER= \ + STORAGE_PASSWORD= \ STORAGE_FILEPATH= ENV LOGGING_LEVEL=verbose \ diff --git a/docker-entrypoint.js b/docker-entrypoint.js index 60be55d..f8912de 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -92,10 +92,10 @@ const config = { expire: STORAGE_EXPIRE_SECONDS, bucket: STORAGE_AWS_BUCKET, region: STORAGE_AWS_REGION, - connectionUrl: `postgres://${STORAGE_PG_USERNAME}:${STORAGE_PG_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`, + connectionUrl: `postgres://${STORAGE_USERNAME}:${STORAGE_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`, db: STORAGE_DB, - user: STORAGE_RETHINK_USER, - password: STORAGE_RETHINK_USER, + user: STORAGE_USERNAME, + password: STORAGE_PASSWORD, path: STORAGE_FILEPATH, }, From a4dc29fb2bc7fff3eb1231e576883d1101c03b0d Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 22:12:54 +0200 Subject: [PATCH 18/22] its supposed to be milliseconds --- Dockerfile | 6 +++--- docker-entrypoint.js | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 34a06e3..dd53efc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,14 +42,14 @@ ENV KEYGENERATOR_TYPE=phonetic \ KEYGENERATOR_KEYSPACE= ENV RATELIMITS_NORMAL_TOTAL_REQUESTS=500\ - RATELIMITS_NORMAL_EVERY_SECONDS=60000 \ + RATELIMITS_NORMAL_EVERY_MILLISECONDS=60000 \ RATELIMITS_WHITELIST_TOTAL_REQUESTS= \ - RATELIMITS_WHITELIST_EVERY_SECONDS= \ + RATELIMITS_WHITELIST_EVERY_MILLISECONDS= \ # comma separated list for the whitelisted \ RATELIMITS_WHITELIST=example1.whitelist,example2.whitelist \ \ RATELIMITS_BLACKLIST_TOTAL_REQUESTS= \ - RATELIMITS_BLACKLIST_EVERY_SECONDS= \ + RATELIMITS_BLACKLIST_EVERY_MILLISECONDS= \ # comma separated list for the blacklisted \ RATELIMITS_BLACKLIST=example1.blacklist,example2.blacklist ENV DOCUMENTS=about=./about.md diff --git a/docker-entrypoint.js b/docker-entrypoint.js index f8912de..05b29da 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -23,12 +23,12 @@ const { KEYGENERATOR_TYPE, KEY_GENERATOR_KEYSPACE, RATE_LIMITS_NORMAL_TOTAL_REQUESTS, - RATE_LIMITS_NORMAL_EVERY_SECONDS, + RATE_LIMITS_NORMAL_EVERY_MILLISECONDS, RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, - RATE_LIMITS_WHITELIST_EVERY_SECONDS, + RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS, RATE_LIMITS_WHITELIST, RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS, - RATE_LIMITS_BLACKLIST_EVERY_SECONDS, + RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS, RATE_LIMITS_BLACKLIST, DOCUMENTS, } = process.env; @@ -64,22 +64,22 @@ const config = { categories: { normal: { totalRequests: RATE_LIMITS_NORMAL_TOTAL_REQUESTS, - every: RATE_LIMITS_NORMAL_EVERY_SECONDS, + every: RATE_LIMITS_NORMAL_EVERY_MILLISECONDS, }, whitelist: - RATE_LIMITS_WHITELIST_EVERY_SECONDS || + RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS || RATE_LIMITS_WHITELIST_TOTAL_REQUESTS ? { totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, - every: RATE_LIMITS_WHITELIST_EVERY_SECONDS, + every: RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS, } : null, blacklist: - RATE_LIMITS_BLACKLIST_EVERY_SECONDS || + RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS || RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS ? { totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS, - every: RATE_LIMITS_BLACKLIST_EVERY_SECONDS, + every: RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS, } : null, }, From bb7b9571a7b7eb1c845d2d4c41f685e7ef2e2574 Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 22:22:08 +0200 Subject: [PATCH 19/22] write some documentation for the Docker solution --- README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/README.md b/README.md index 23ca077..1cb06f7 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,88 @@ your bucket: } ``` +## Docker + +### Build image + +```bash +docker build --tag haste-server . +``` + +### Run container + +For this example we will run haste-server, and connect it to a redis server + +```bash +docker run --name haste-server-container --env STORAGE_TYPE=redis --env STORAGE_HOST=redis-server --env STORAGE_PORT=6379 haste-server +``` + +### Use docker-compose example + +There is an example `docker-compose.yml` which runs haste-server together with memcached + +```bash +docker-compose up +``` + +### Configuration + +The docker image is configured using environmental variables as you can see in the example above. + +Here is a list of all the environment variables + +### Storage + +| Name | Default value | Description | +| :--------------------: | :-----------: | :-----------------------------------------------------------------------------------------------------------: | +| STORAGE_TYPE | memcached | Type of storage . Accepted values: "memcached","redis","postgres","rethinkdb", "amazon-s3", and "file" | +| STORAGE_HOST | 127.0.0.1 | Storage host. Applicable for types: memcached, redis, postgres, and rethinkdb | +| STORAGE_PORT | 11211 | Port on the storage host. Applicable for types: memcached, redis, postgres, and rethinkdb | +| STORAGE_EXPIRE_SECONDS | 2592000 | Number of seconds to expire keys in. Applicable for types. Redis, postgres, memcached. `expire` option to the | +| STORAGE_DB | 2 | The name of the database. Applicable for redis, postgres, and rethinkdb | +| STORAGE_PASSWORD | | Password for database. Applicable for redis, postges, rethinkdb . | +| STORAGE_USERNAME | | Database username. Applicable for postgres, and rethinkdb | +| STORAGE_AWS_BUCKET | | Applicable for amazon-s3. This is the name of the S3 bucket | +| STORAGE_AWS_REGION | | Applicable for amazon-s3. The region in which the bucket is located | +| STORAGE_FILEPATH | | Path to file to save data to. Applicable for type file | + +### Logging + +| Name | Default value | Description | +| :---------------: | :-----------: | :---------: | +| LOGGING_LEVEL | verbose | | +| LOGGING_TYPE= | Console | +| LOGGING_COLORIZE= | true | + +### Basics + +| Name | Default value | Description | +| :----------------------: | :--------------: | :---------------------------------------------------------------------------------------: | +| HOST | 0.0.0.0 | The hostname which the server answers on | +| PORT | 7777 | The port on which the server is running | +| KEY_LENGTH | 10 | the length of the keys to user | +| MAX_LENGTH | 400000 | maximum length of a paste | +| STATIC_MAX_AGE | 86400 | max age for static assets | +| RECOMPRESS_STATIC_ASSETS | true | whether or not to compile static js assets | +| KEYGENERATOR_TYPE | phonetic | Type of key generator. Acceptable values: "phonetic", or "random" | +| KEYGENERATOR_KEYSPACE | | keySpace argument is a string of acceptable characters | +| DOCUMENTS | about=./about.md | Comma separated list of static documents to serve. ex: \n about=./about.md,home=./home.md | + +### Rate limits + +| Name | Default value | Description | +| :----------------------------------: | :-----------------------------------: | :--------------------------------------------------------------------------------------: | +| RATELIMITS_NORMAL_TOTAL_REQUESTS | 500 | By default anyone uncategorized will be subject to 500 requests in the defined timespan. | +| RATELIMITS_NORMAL_EVERY_MILLISECONDS | 60000 | The timespan to allow the total requests for uncategorized users | +| RATELIMITS_WHITELIST_TOTAL_REQUESTS | | By default client names in the whitelist will not have their requests limited. | +| RATELIMITS_WHITELIST_EVERY_SECONDS | | By default client names in the whitelist will not have their requests limited. | +| RATELIMITS_WHITELIST | example1.whitelist,example2.whitelist | Comma separated list of the clients which are in the whitelist pool | +| RATELIMITS_BLACKLIST_TOTAL_REQUESTS | | By default client names in the blacklist will be subject to 0 requests per hours. | +| RATELIMITS_BLACKLIST_EVERY_SECONDS | | By default client names in the blacklist will be subject to 0 requests per hours | +| RATELIMITS_BLACKLIST | example1.blacklist,example2.blacklist | Comma separated list of the clients which are in the blacklistpool. | + + + ## Author John Crepezzi From bae6387bb71bd3bffd28bbafc2ff38c3f665e84e Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 22:25:10 +0200 Subject: [PATCH 20/22] forgot to rename some vars --- docker-entrypoint.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-entrypoint.js b/docker-entrypoint.js index 05b29da..5afff14 100644 --- a/docker-entrypoint.js +++ b/docker-entrypoint.js @@ -12,10 +12,8 @@ const { STORAGE_DB, STORAGE_AWS_BUCKET, STORAGE_AWS_REGION, - STORAGE_PG_PASSWORD, - STORAGE_PG_USERNAME, - STORAGE_RETHINK_USER, - STORAGE_RETHINK_PASSWORD, + STORAGE_PASSWORD, + STORAGE_USERNAME, STORAGE_FILEPATH, LOGGING_LEVEL, LOGGING_TYPE, From 139df62ec4c3ca959db327e265751a740257f83b Mon Sep 17 00:00:00 2001 From: emil-lengman Date: Sat, 22 Aug 2020 22:27:05 +0200 Subject: [PATCH 21/22] add newline to stop github complaining --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 8e4e5a1..0b089d8 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -6,4 +6,4 @@ set -e node ./docker-entrypoint.js > ./config.js -exec "$@" \ No newline at end of file +exec "$@" From 16bce4c83d073c7a9695a63efac3d0966ed4556e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2020 19:40:54 +0000 Subject: [PATCH 22/22] Bump bl from 4.0.2 to 4.0.3 Bumps [bl](https://github.com/rvagg/bl) from 4.0.2 to 4.0.3. - [Release notes](https://github.com/rvagg/bl/releases) - [Commits](https://github.com/rvagg/bl/compare/v4.0.2...v4.0.3) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index cd5d970..423a8d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,9 +29,9 @@ "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "bl": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", - "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4",