Merge upstream changes

Fixes #376 + .dockerignore improvement

.dockerignore

@@ -0,0 +1,8 @@
+Dockerfile
+.git
+npm-debug.log
+node_modules
+*.swp
+*.swo
+data
+*.DS_Store

.gitignore

@@ -4,3 +4,4 @@ node_modules
 *.swo
 data
 *.DS_Store
+config.json

Dockerfile

@@ -0,0 +1,68 @@
+FROM node:14.8.0-stretch
+
+RUN mkdir -p /usr/src/app && \
+    chown node:node /usr/src/app
+
+USER node:node
+
+WORKDIR /usr/src/app
+
+COPY --chown=node:node . .
+
+RUN npm install && \
+    npm install redis@0.8.1 && \
+    npm install pg@4.1.1 && \
+    npm install memcached@2.2.2 && \
+    npm install aws-sdk@2.738.0 && \
+    npm install rethinkdbdash@2.3.31
+
+ENV STORAGE_TYPE=memcached \
+    STORAGE_HOST=127.0.0.1 \
+    STORAGE_PORT=11211\
+    STORAGE_EXPIRE_SECONDS=2592000\
+    STORAGE_DB=2 \
+    STORAGE_AWS_BUCKET= \
+    STORAGE_AWS_REGION= \
+    STORAGE_USENAME= \
+    STORAGE_PASSWORD= \
+    STORAGE_FILEPATH=
+
+ENV LOGGING_LEVEL=verbose \
+    LOGGING_TYPE=Console \
+    LOGGING_COLORIZE=true
+
+ENV HOST=0.0.0.0\
+    PORT=7777\
+    KEY_LENGTH=10\
+    MAX_LENGTH=400000\
+    STATIC_MAX_AGE=86400\
+    RECOMPRESS_STATIC_ASSETS=true
+
+ENV KEYGENERATOR_TYPE=phonetic \
+    KEYGENERATOR_KEYSPACE=
+
+ENV RATELIMITS_NORMAL_TOTAL_REQUESTS=500\
+    RATELIMITS_NORMAL_EVERY_MILLISECONDS=60000 \
+    RATELIMITS_WHITELIST_TOTAL_REQUESTS= \
+    RATELIMITS_WHITELIST_EVERY_MILLISECONDS=  \
+    # comma separated list for the whitelisted \
+    RATELIMITS_WHITELIST=example1.whitelist,example2.whitelist \
+    \
+    RATELIMITS_BLACKLIST_TOTAL_REQUESTS= \
+    RATELIMITS_BLACKLIST_EVERY_MILLISECONDS= \
+    # comma separated list for the blacklisted \
+    RATELIMITS_BLACKLIST=example1.blacklist,example2.blacklist
+ENV DOCUMENTS=about=./about.md
+
+EXPOSE ${PORT}
+STOPSIGNAL SIGINT
+ENTRYPOINT [ "bash", "docker-entrypoint.sh" ]
+
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s \
+    --retries=3 CMD [ "sh", "-c", "echo -n 'curl localhost:7777... '; \
+    (\
+        curl -sf localhost:7777 > /dev/null\
+    ) && echo OK || (\
+        echo Fail && exit 2\
+    )"]
+CMD ["npm", "start"]

README.md

@@ -31,7 +31,7 @@ STDOUT.  Check the README there for more details and usages.
 1.  Download the package, and expand it
 2.  Explore the settings inside of config.js, but the defaults should be good
 3.  `npm install`
-4.  `npm start`
+4.  `npm start` (you may specify an optional `<config-path>` as well)
 
 ## Settings
 
@@ -52,7 +52,7 @@ STDOUT.  Check the README there for more details and usages.
 
 When present, the `rateLimits` option enables built-in rate limiting courtesy
 of `connect-ratelimit`.  Any of the options supported by that library can be
-used and set in `config.json`.
+used and set in `config.js`.
 
 See the README for [connect-ratelimit](https://github.com/dharmafly/connect-ratelimit)
 for more information!
@@ -154,6 +154,28 @@ or post.
 
 All of which are optional except `type` with very logical default values.
 
+### MongoDB
+
+To use mongodb storage you must install the 'mongodb' package in npm
+
+`npm install mongodb`
+
+Once you've done that, your config section should look like:
+
+``` json
+{
+  "type": "mongo",
+  "connectionUrl": "mongodb://localhost:27017/database"
+}
+```
+
+You can also just set the environment variable for `DATABASE_URL` to your database connection url.
+
+Unlike with postgres you do NOT have to create the table in your mongo database prior to running.
+
+You can also set an `expire` option to the number of seconds to expire keys in.
+This is off by default, but will constantly kick back expirations on each view or post.
+
 ### Memcached
 
 To use memcache storage you must install the `memcached` package via npm
@@ -198,6 +220,140 @@ Also, you must create an `uploads` table, which will store all the data for uplo
 
 You can optionally add the `user` and `password` properties to use a user system.
 
+### Google Datastore
+
+To use the Google Datastore storage system, you must install the `@google-cloud/datastore` package via npm
+
+`npm install @google-cloud/datastore`
+
+Once you've done that, your config section should look like this:
+
+``` json
+{
+  "type": "google-datastore"
+}
+```
+
+Authentication is handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started), by providing authentication details to the GOOGLE_APPLICATION_CREDENTIALS environmental variable.
+
+### Amazon S3
+
+To use [Amazon S3](https://aws.amazon.com/s3/) as a storage system, you must
+install the `aws-sdk` package via npm:
+
+`npm install aws-sdk`
+
+Once you've done that, your config section should look like this:
+
+```json
+{
+  "type": "amazon-s3",
+  "bucket": "your-bucket-name",
+  "region": "us-east-1"
+}
+```
+
+Authentication is handled automatically by the client. Check
+[Amazon's documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html)
+for more information. You will need to grant your role these permissions to
+your bucket:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetObject",
+                "s3:PutObject"
+            ],
+            "Effect": "Allow",
+            "Resource": "arn:aws:s3:::your-bucket-name-goes-here/*"
+        }
+    ]
+}
+```
+
+## Docker
+
+### Build image
+
+```bash
+docker build --tag haste-server .
+```
+
+### Run container
+
+For this example we will run haste-server, and connect it to a redis server
+
+```bash
+docker run --name haste-server-container --env STORAGE_TYPE=redis --env STORAGE_HOST=redis-server --env STORAGE_PORT=6379 haste-server
+```
+
+### Use docker-compose example
+
+There is an example `docker-compose.yml` which runs haste-server together with memcached
+
+```bash
+docker-compose up
+```
+
+### Configuration
+
+The docker image is configured using environmental variables as you can see in the example above.
+
+Here is a list of all the environment variables
+
+### Storage
+
+|          Name          | Default value |                                                  Description                                                  |
+| :--------------------: | :-----------: | :-----------------------------------------------------------------------------------------------------------: |
+|      STORAGE_TYPE      |   memcached   |    Type of storage . Accepted values: "memcached","redis","postgres","rethinkdb", "amazon-s3", and "file"     |
+|      STORAGE_HOST      |   127.0.0.1   |                 Storage host. Applicable for types: memcached, redis, postgres, and rethinkdb                 |
+|      STORAGE_PORT      |     11211     |           Port on the storage host. Applicable for types: memcached, redis, postgres, and rethinkdb           |
+| STORAGE_EXPIRE_SECONDS |    2592000    | Number of seconds to expire keys in. Applicable for types. Redis, postgres, memcached. `expire` option to the |
+|       STORAGE_DB       |       2       |                    The name of the database. Applicable for redis, postgres, and rethinkdb                    |
+|    STORAGE_PASSWORD    |               |                       Password for database. Applicable for redis, postges, rethinkdb .                       |
+|    STORAGE_USERNAME    |               |                           Database username. Applicable for postgres, and rethinkdb                           |
+|   STORAGE_AWS_BUCKET   |               |                          Applicable for amazon-s3. This is the name of the S3 bucket                          |
+|   STORAGE_AWS_REGION   |               |                      Applicable for amazon-s3. The region in which the bucket is located                      |
+|    STORAGE_FILEPATH    |               |                            Path to file to save data to. Applicable for type file                             |
+
+### Logging
+
+|       Name        | Default value | Description |
+| :---------------: | :-----------: | :---------: |
+|   LOGGING_LEVEL   |    verbose    |             |
+|   LOGGING_TYPE=   |    Console    |
+| LOGGING_COLORIZE= |     true      |
+
+### Basics
+
+|           Name           |  Default value   |                                        Description                                        |
+| :----------------------: | :--------------: | :---------------------------------------------------------------------------------------: |
+|           HOST           |     0.0.0.0      |                         The hostname which the server answers on                          |
+|           PORT           |       7777       |                          The port on which the server is running                          |
+|        KEY_LENGTH        |        10        |                              the length of the keys to user                               |
+|        MAX_LENGTH        |      400000      |                                 maximum length of a paste                                 |
+|      STATIC_MAX_AGE      |      86400       |                                 max age for static assets                                 |
+| RECOMPRESS_STATIC_ASSETS |       true       |                        whether or not to compile static js assets                         |
+|    KEYGENERATOR_TYPE     |     phonetic     |             Type of key generator. Acceptable values: "phonetic", or "random"             |
+|  KEYGENERATOR_KEYSPACE   |                  |                  keySpace argument is a string of acceptable characters                   |
+|        DOCUMENTS         | about=./about.md | Comma separated list of static documents to serve. ex: \n about=./about.md,home=./home.md |
+
+### Rate limits
+
+|                 Name                 |             Default value             |                                       Description                                        |
+| :----------------------------------: | :-----------------------------------: | :--------------------------------------------------------------------------------------: |
+|   RATELIMITS_NORMAL_TOTAL_REQUESTS   |                  500                  | By default anyone uncategorized will be subject to 500 requests in the defined timespan. |
+| RATELIMITS_NORMAL_EVERY_MILLISECONDS |                 60000                 |             The timespan to allow the total requests for uncategorized users             |
+| RATELIMITS_WHITELIST_TOTAL_REQUESTS  |                                       |      By default client names in the whitelist will not have their requests limited.      |
+|  RATELIMITS_WHITELIST_EVERY_SECONDS  |                                       |      By default client names in the whitelist will not have their requests limited.      |
+|         RATELIMITS_WHITELIST         | example1.whitelist,example2.whitelist |           Comma separated list of the clients which are in the whitelist pool            |
+| RATELIMITS_BLACKLIST_TOTAL_REQUESTS  |                                       |    By default client names in the blacklist will be subject to 0 requests per hours.     |
+|  RATELIMITS_BLACKLIST_EVERY_SECONDS  |                                       |     By default client names in the blacklist will be subject to 0 requests per hours     |
+|         RATELIMITS_BLACKLIST         | example1.blacklist,example2.blacklist |           Comma separated list of the clients which are in the blacklistpool.            |
+
 ## Author
 
 John Crepezzi <john.crepezzi@gmail.com>

about.md

@@ -15,45 +15,19 @@ To make a new entry, click "New" (or type 'control + n')
 
 ## From the Console
 
-Most of the time I want to show you some text, it's coming from my current
-console session.  We should make it really easy to take code from the console
-and send it to people.
+[bin-client](git.webionite.com/ceda_ei/bin-client)
 
-`cat something | haste` # https://hastebin.com/1238193
-
-You can even take this a step further, and cut out the last step of copying the
-URL with:
-
-* osx: `cat something | haste | pbcopy`
-* linux: `cat something | haste | xsel`
-* windows: check out [WinHaste](https://github.com/ajryan/WinHaste)
-
-After running that, the STDOUT output of `cat something` will show up at a URL
-which has been conveniently copied to your clipboard.
-
-That's all there is to that, and you can install it with `gem install haste`
-right now.
-  * osx: you will need to have an up to date version of Xcode
-  * linux: you will need to have rubygems and ruby-devel installed
-
-## Duration
-
-Pastes will stay for 30 days from their last view.  They may be removed earlier
-and without notice.
-
-## Privacy
-
-While the contents of hastebin.com are not directly crawled by any search robot
-that obeys "robots.txt", there should be no great expectation of privacy.  Post
-things at your own risk. Not responsible for any loss of data or removed
-pastes.
+Add the following to your bashrc/zshrc
+```
+export MKR_BIN='https://bin.webionite.com/'
+export HASTEBIN=1
+```
 
 ## Open Source
 
 Haste can easily be installed behind your network, and it's all open source!
 
-* [haste-client](https://github.com/seejohnrun/haste-client)
-* [haste-server](https://github.com/seejohnrun/haste-server)
+* [haste-server](https://git.webionite.com/Webionite/haste-server)
 
 ## Author
 

docker-compose.yaml

@@ -0,0 +1,19 @@
+version: '3.0'
+services:
+  haste-server:
+    build: .
+    networks:
+      - db-network
+    environment:
+      - STORAGE_TYPE=memcached
+      - STORAGE_HOST=memcached
+      - STORAGE_PORT=11211
+    ports:
+      - 7777:7777
+  memcached:
+    image: memcached:latest
+    networks:
+      - db-network
+
+networks:
+  db-network:

docker-entrypoint.js

@@ -0,0 +1,108 @@
+const {
+  HOST,
+  PORT,
+  KEY_LENGTH,
+  MAX_LENGTH,
+  STATIC_MAX_AGE,
+  RECOMPRESS_STATIC_ASSETS,
+  STORAGE_TYPE,
+  STORAGE_HOST,
+  STORAGE_PORT,
+  STORAGE_EXPIRE_SECONDS,
+  STORAGE_DB,
+  STORAGE_AWS_BUCKET,
+  STORAGE_AWS_REGION,
+  STORAGE_PASSWORD,
+  STORAGE_USERNAME,
+  STORAGE_FILEPATH,
+  LOGGING_LEVEL,
+  LOGGING_TYPE,
+  LOGGING_COLORIZE,
+  KEYGENERATOR_TYPE,
+  KEY_GENERATOR_KEYSPACE,
+  RATE_LIMITS_NORMAL_TOTAL_REQUESTS,
+  RATE_LIMITS_NORMAL_EVERY_MILLISECONDS,
+  RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
+  RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS,
+  RATE_LIMITS_WHITELIST,
+  RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS,
+  RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS,
+  RATE_LIMITS_BLACKLIST,
+  DOCUMENTS,
+} = process.env;
+
+const config = {
+  host: HOST,
+  port: PORT,
+
+  keyLength: KEY_LENGTH,
+
+  maxLength: MAX_LENGTH,
+
+  staticMaxAge: STATIC_MAX_AGE,
+
+  recompressStaticAssets: RECOMPRESS_STATIC_ASSETS,
+
+  logging: [
+    {
+      level: LOGGING_LEVEL,
+      type: LOGGING_TYPE,
+      colorize: LOGGING_COLORIZE,
+    },
+  ],
+
+  keyGenerator: {
+    type: KEYGENERATOR_TYPE,
+    keyspace: KEY_GENERATOR_KEYSPACE,
+  },
+
+  rateLimits: {
+    whitelist: RATE_LIMITS_WHITELIST ? RATE_LIMITS_WHITELIST.split(",") : [],
+    blacklist: RATE_LIMITS_BLACKLIST ? RATE_LIMITS_BLACKLIST.split(",") : [],
+    categories: {
+      normal: {
+        totalRequests: RATE_LIMITS_NORMAL_TOTAL_REQUESTS,
+        every: RATE_LIMITS_NORMAL_EVERY_MILLISECONDS,
+      },
+      whitelist:
+        RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS ||
+        RATE_LIMITS_WHITELIST_TOTAL_REQUESTS
+          ? {
+              totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
+              every: RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS,
+            }
+          : null,
+      blacklist:
+        RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS ||
+        RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS
+          ? {
+              totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
+              every: RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS,
+            }
+          : null,
+    },
+  },
+
+  storage: {
+    type: STORAGE_TYPE,
+    host: STORAGE_HOST,
+    port: STORAGE_PORT,
+    expire: STORAGE_EXPIRE_SECONDS,
+    bucket: STORAGE_AWS_BUCKET,
+    region: STORAGE_AWS_REGION,
+    connectionUrl: `postgres://${STORAGE_USERNAME}:${STORAGE_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`,
+    db: STORAGE_DB,
+    user: STORAGE_USERNAME,
+    password: STORAGE_PASSWORD,
+    path: STORAGE_FILEPATH,
+  },
+
+  documents: DOCUMENTS
+    ? DOCUMENTS.split(",").reduce((acc, item) => {
+        const keyAndValueArray = item.replace(/\s/g, "").split("=");
+        return { ...acc, [keyAndValueArray[0]]: keyAndValueArray[1] };
+      }, {})
+    : null,
+};
+
+console.log(JSON.stringify(config));

docker-entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# We use this file to translate environmental variables to .env files used by the application
+
+set -e
+
+node ./docker-entrypoint.js > ./config.js
+
+exec "$@"

lib/document_handler.js

@@ -16,33 +16,55 @@ var DocumentHandler = function(options) {
 DocumentHandler.defaultKeyLength = 10;
 
 // Handle retrieving a document
-DocumentHandler.prototype.handleGet = function(key, response, skipExpire) {
+DocumentHandler.prototype.handleGet = function(request, response, config) {
+  const key = request.params.id.split('.')[0];
+  const skipExpire = !!config.documents[key];
+
   this.store.get(key, function(ret) {
     if (ret) {
       winston.verbose('retrieved document', { key: key });
       response.writeHead(200, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ data: ret, key: key }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ data: ret, key: key }));
+      }
     }
     else {
       winston.warn('document not found', { key: key });
       response.writeHead(404, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ message: 'Document not found.' }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ message: 'Document not found.' }));
+      }
     }
   }, skipExpire);
 };
 
 // Handle retrieving the raw version of a document
-DocumentHandler.prototype.handleRawGet = function(key, response, skipExpire) {
+DocumentHandler.prototype.handleRawGet = function(request, response, config) {
+  const key = request.params.id.split('.')[0];
+  const skipExpire = !!config.documents[key];
+
   this.store.get(key, function(ret) {
     if (ret) {
       winston.verbose('retrieved raw document', { key: key });
       response.writeHead(200, { 'content-type': 'text/plain; charset=UTF-8' });
-      response.end(ret);
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(ret);
+      }
     }
     else {
       winston.warn('raw document not found', { key: key });
       response.writeHead(404, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ message: 'Document not found.' }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ message: 'Document not found.' }));
+      }
     }
   }, skipExpire);
 };

lib/document_stores/amazon-s3.js

@@ -0,0 +1,56 @@
+/*global require,module,process*/
+
+var AWS = require('aws-sdk');
+var winston = require('winston');
+
+var AmazonS3DocumentStore = function(options) {
+  this.expire = options.expire;
+  this.bucket = options.bucket;
+  this.client = new AWS.S3({region: options.region});
+};
+
+AmazonS3DocumentStore.prototype.get = function(key, callback, skipExpire) {
+  var _this = this;
+
+  var req = {
+    Bucket: _this.bucket,
+    Key: key
+  };
+
+  _this.client.getObject(req, function(err, data) {
+    if(err) {
+      callback(false);
+    }
+    else {
+      callback(data.Body.toString('utf-8'));
+      if (_this.expire && !skipExpire) {
+        winston.warn('amazon s3 store cannot set expirations on keys');
+      }
+    }
+  });
+}
+
+AmazonS3DocumentStore.prototype.set = function(key, data, callback, skipExpire) {
+  var _this = this;
+
+  var req = {
+    Bucket: _this.bucket,
+    Key: key,
+    Body: data,
+    ContentType: 'text/plain'
+  };
+
+  _this.client.putObject(req, function(err, data) {
+    if (err) {
+      callback(false);
+    }
+    else {
+      callback(true);
+      if (_this.expire && !skipExpire) {
+        winston.warn('amazon s3 store cannot set expirations on keys');
+      }
+    }
+  });
+}
+
+module.exports = AmazonS3DocumentStore;

lib/document_stores/google-datastore.js

@@ -0,0 +1,89 @@
+/*global require,module,process*/
+
+const Datastore = require('@google-cloud/datastore');
+const winston = require('winston');
+
+class GoogleDatastoreDocumentStore {
+
+  // Create a new store with options
+  constructor(options) {
+    this.kind = "Haste";
+    this.expire = options.expire;
+    this.datastore = new Datastore();
+  }
+
+  // Save file in a key
+  set(key, data, callback, skipExpire) {
+    var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(Date.now() + this.expire * 1000);
+
+    var taskKey = this.datastore.key([this.kind, key])
+    var task = {
+      key: taskKey,
+      data: [
+        {
+          name: 'value',
+          value: data,
+          excludeFromIndexes: true
+        },
+        {
+          name: 'expiration',
+          value: expireTime
+        }
+      ]
+    };
+
+    this.datastore.insert(task).then(() => {
+      callback(true);
+    })
+    .catch(err => {
+      callback(false);
+    });
+  }
+
+  // Get a file from a key
+  get(key, callback, skipExpire) {
+    var taskKey = this.datastore.key([this.kind, key])
+
+    this.datastore.get(taskKey).then((entity) => {
+      if (skipExpire || entity[0]["expiration"] == null) {
+        callback(entity[0]["value"]);
+      }
+      else {
+        // check for expiry
+        if (entity[0]["expiration"] < new Date()) {
+          winston.info("document expired", {key: key, expiration: entity[0]["expiration"], check: new Date(null)});
+          callback(false);
+        }
+        else {
+          // update expiry
+          var task = {
+            key: taskKey,
+            data: [
+              {
+                name: 'value',
+                value: entity[0]["value"],
+                excludeFromIndexes: true
+              },
+              {
+                name: 'expiration',
+                value: new Date(Date.now() + this.expire * 1000)
+              }
+            ]
+          };
+          this.datastore.update(task).then(() => {
+          })
+          .catch(err => {
+            winston.error("failed to update expiration", {error: err});
+          });
+          callback(entity[0]["value"]);
+        }
+      }
+    })
+    .catch(err => {
+        winston.error("Error retrieving value from Google Datastore", {error: err});
+        callback(false);
+    });
+  }
+}
+
+module.exports = GoogleDatastoreDocumentStore;

lib/document_stores/memcached.js

@@ -26,7 +26,7 @@ class MemcachedDocumentStore {
 
   // Save file in a key
   set(key, data, callback, skipExpire) {
-    this.client.set(key, data, skipExpire ? 0 : this.expire, (error) => {
+    this.client.set(key, data, skipExpire ? 0 : this.expire || 0, (error) => {
       callback(!error);
     });
   }
@@ -34,10 +34,12 @@ class MemcachedDocumentStore {
   // Get a file from a key
   get(key, callback, skipExpire) {
     this.client.get(key, (error, data) => {
-      callback(error ? false : data);
+      const value = error ? false : data;
+
+      callback(value);
 
       // Update the key so that the expiration is pushed forward
-      if (!skipExpire) {
+      if (value && !skipExpire) {
         this.set(key, data, (updateSucceeded) => {
           if (!updateSucceeded) {
             winston.error('failed to update expiration on GET', {key});

lib/document_stores/mongo.js

@@ -0,0 +1,88 @@
+
+
+var MongoClient = require('mongodb').MongoClient,
+    winston = require('winston');
+
+var MongoDocumentStore = function (options) {
+    this.expire = options.expire;
+    this.connectionUrl = process.env.DATABASE_URl || options.connectionUrl;
+};
+
+MongoDocumentStore.prototype.set = function (key, data, callback, skipExpire) {
+    var now = Math.floor(new Date().getTime() / 1000),
+        that = this;
+
+    this.safeConnect(function (err, db) {
+        if (err)
+            return callback(false);
+        
+        db.collection('entries').update({
+            'entry_id': key,
+            $or: [
+                { expiration: -1 },
+                { expiration: { $gt: now } }
+            ]
+        }, {
+            'entry_id': key,
+            'value': data,
+            'expiration': that.expire && !skipExpire ? that.expire + now : -1
+        }, {
+            upsert: true
+        }, function (err, existing) {
+            if (err) {
+                winston.error('error persisting value to mongodb', { error: err });
+                return callback(false);
+            }
+
+            callback(true);
+        });
+    });
+};
+
+MongoDocumentStore.prototype.get = function (key, callback, skipExpire) {
+    var now = Math.floor(new Date().getTime() / 1000),
+        that = this;
+
+    this.safeConnect(function (err, db) {
+        if (err)
+            return callback(false);
+        
+        db.collection('entries').findOne({
+            'entry_id': key,
+            $or: [
+                { expiration: -1 },
+                { expiration: { $gt: now } }
+            ]
+        }, function (err, entry) {
+            if (err) {
+                winston.error('error persisting value to mongodb', { error: err });
+                return callback(false);
+            }
+
+            callback(entry === null ? false : entry.value);
+
+            if (entry !== null && entry.expiration !== -1 && that.expire && !skipExpire) {
+                db.collection('entries').update({
+                    'entry_id': key
+                }, {
+                    $set: {
+                        'expiration': that.expire + now
+                    }
+                }, function (err, result) { });
+            }
+        });
+    });
+};
+
+MongoDocumentStore.prototype.safeConnect = function (callback) {
+    MongoClient.connect(this.connectionUrl, function (err, db) {
+        if (err) {
+            winston.error('error connecting to mongodb', { error: err });
+            callback(err);
+        } else {
+            callback(undefined, db);
+        }
+    });
+};
+
+module.exports = MongoDocumentStore;

lib/document_stores/postgres.js

@@ -1,14 +1,16 @@
 /*global require,module,process*/
 
-var postgres = require('pg');
 var winston = require('winston');
+const {Pool} = require('pg');
 
 // create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));
 
 // A postgres document store
 var PostgresDocumentStore = function (options) {
   this.expireJS = options.expire;
-  this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl;
+
+  const connectionString = process.env.DATABASE_URL || options.connectionUrl;
+  this.pool = new Pool({connectionString});
 };
 
 PostgresDocumentStore.prototype = {
@@ -64,16 +66,15 @@ PostgresDocumentStore.prototype = {
 
   // A connection wrapper
   safeConnect: function (callback) {
-    postgres.connect(this.connectionUrl, function (err, client, done) {
-      if (err) {
-        winston.error('error connecting to postgres', { error: err });
-        callback(err);
+    this.pool.connect((error, client, done) => {
+      if (error) {
+        winston.error('error connecting to postgres', {error});
+        callback(error);
       } else {
         callback(undefined, client, done);
       }
     });
   }
-
 };
 
 module.exports = PostgresDocumentStore;

package.json

@@ -14,25 +14,21 @@
   },
   "main": "haste",
   "dependencies": {
+    "busboy": "0.2.4",
+    "connect": "^3.7.0",
     "connect-ratelimit": "0.0.7",
     "connect-route": "0.1.5",
-    "connect": "3.4.1",
-    "st": "1.1.0",
-    "winston": "0.6.2",
-    "redis-url": "0.1.0",
+    "pg": "^8.0.0",
     "redis": "0.8.1",
+    "redis-url": "0.1.0",
+    "st": "^2.0.0",
     "uglify-js": "3.1.6",
-    "busboy": "0.2.4",
-    "pg": "4.1.1"
+    "winston": "^2.0.0"
   },
   "devDependencies": {
-    "mocha": "^4.0.1"
+    "mocha": "^8.1.3"
   },
   "bundledDependencies": [],
-  "engines": {
-    "node": "8.1.4",
-    "npm": "5.2.0"
-  },
   "bin": {
     "haste-server": "./server.js"
   },

sample.config.json

@@ -33,10 +33,7 @@
   },
 
   "storage": {
-    "type": "memcached",
-    "host": "127.0.0.1",
-    "port": 11211,
-    "expire": 2592000
+    "type": "file"
   },
 
   "documents": {

server.js

@@ -1,3 +1,4 @@
+// vim: set ts=2 sw=2 sts=2 et:
 var http = require('http');
 var fs = require('fs');
 
@@ -11,7 +12,7 @@ var connect_rate_limit = require('connect-ratelimit');
 var DocumentHandler = require('./lib/document_handler');
 
 // Load the configuration and set some defaults
-var config = JSON.parse(fs.readFileSync('./config.js', 'utf8'));
+var config = require('./config.json');
 config.port = process.env.PORT || config.port || 7777;
 config.host = process.env.HOST || config.host || 'localhost';
 
@@ -106,23 +107,30 @@ if (config.rateLimits) {
   app.use(connect_rate_limit(config.rateLimits));
 }
 
+function raw(request, response) {
+  return documentHandler.handleRawGet(request, response, config);
+}
+
 // first look at API calls
 app.use(route(function(router) {
   // get raw documents - support getting with extension
-  router.get('/raw/:id', function(request, response) {
-    var key = request.params.id.split('.')[0];
-    var skipExpire = !!config.documents[key];
-    return documentHandler.handleRawGet(key, response, skipExpire);
-  });
+  router.get('/raw/:id', raw);
+  router.get('/:id/raw', raw);
+  router.head('/raw/:id', raw);
+  router.head('/:id/raw', raw);
   // add documents
+
   router.post('/documents', function(request, response) {
     return documentHandler.handlePost(request, response);
   });
+
   // get documents
   router.get('/documents/:id', function(request, response) {
-    var key = request.params.id.split('.')[0];
-    var skipExpire = !!config.documents[key];
-    return documentHandler.handleGet(key, response, skipExpire);
+    return documentHandler.handleGet(request, response, config);
+  });
+
+  router.head('/documents/:id', function(request, response) {
+    return documentHandler.handleGet(request, response, config);
   });
 }));
 
@@ -138,8 +146,12 @@ app.use(connect_st({
 // so route it back to /
 app.use(route(function(router) {
   router.get('/:id', function(request, response, next) {
-    request.sturl = '/';
-    next();
+    if (request.headers.accept && request.headers.accept.includes('html')) {
+      request.sturl = '/';
+      next();
+    } else {
+      return raw(request, response);
+    }
   });
 }));
 

static/application.css

@@ -1,5 +1,5 @@
 body {
-	background: #002B36;
+	background: #fcfcfc;
 	padding: 20px 50px;
 	margin: 0px;
 }
@@ -9,7 +9,7 @@ body {
 textarea {
 	background: transparent;
 	border: 0px;
-	color: #fff;
+	color: #000;
 	padding: 0px;
 	width: 100%;
 	height: 100%;
@@ -17,20 +17,23 @@ textarea {
 	outline: none;
 	resize: none;
 	font-size: 13px;
+	margin-top: 0;
+	margin-bottom: 0;
 }
 
 /* the line numbers */
 
 #linenos {
-  color: #7d7d7d;
+	color: #7d7d7d;
 	z-index: -1000;
 	position: absolute;
-	top: 20px;
+	/* top: 20px; */
 	left: 0px;
 	width: 30px; /* 30 to get 20 away from box */
 	font-size: 13px;
 	font-family: monospace;
 	text-align: right;
+	user-select: none;
 }
 
 /* code box when locked */
@@ -42,7 +45,8 @@ textarea {
 	border: 0px;
 	outline: none;
 	font-size: 13px;
-  overflow: inherit;
+	overflow: inherit;
+	background: #fcfcfc;
 }
 
 #box code {
@@ -53,32 +57,35 @@ textarea {
 /* key */
 
 #key {
-	position: fixed;
-	top: 0px;
-	right: 0px;
-	z-index: +1000; /* watch out */
+	display: flex;
+	justify-content: space-between;
+	width: 100%;
+	position: sticky;
+	flex-wrap: row;
 }
-
 #box1 {
 	padding: 5px;
 	text-align: center;
-	background: #00222b;
+	background: #fcfcfc;
 }
 
 #box2 {
-	background: #08323c;
+	display: flex;
+	justify-content: right;
+	background: #fcfcfc;
 	font-size: 0px;
 	padding: 0px 5px;
 }
 
-#box1 a.logo, #box1 a.logo:visited {
+a.logo, a.logo:visited {
 	display: inline-block;
 	background: url(logo.png);
 	width: 126px;
+	min-width: 126px;
 	height: 42px;
 }
 
-#box1 a.logo:hover {
+a.logo:hover {
 	background-position: 0 bottom;
 }
 
@@ -112,18 +119,26 @@ textarea {
 }
 
 #box3, #messages li {
-	background: #173e48;
+	position: absolute;
+	right: 100px;
+	background: #fcfcfc;
 	font-family: Helvetica, sans-serif;
 	font-size: 12px;
 	line-height: 14px;
 	padding: 10px 15px;
+	user-select: none;
 }
 
-#box3 .label, #messages li {
-	color: #fff;
+#box3 .label {
+	color: #000;
 	font-weight: bold;
 }
 
+#messages li {
+	color: #FFF;
+	right: 50px;
+}
+
 #box3 .shortcut {
 	color: #c4dce3;
 	font-weight: normal;
@@ -151,16 +166,12 @@ textarea {
 #box2 .button-picture{ border-width: 0; font-size: inherit; }
 
 #messages {
-	position:fixed;
-	top:0px;
-	right:138px;
 	margin:0;
 	padding:0;
-	width:400px;
 }
 
 #messages li {
-	background:rgba(23,62,72,0.8);
+	background:rgba(252,252,252,0.8);
 	margin:0 auto;
 	list-style:none;
 }

static/application.js

@@ -64,7 +64,7 @@ haste_document.prototype.save = function(data, callback) {
     type: 'post',
     data: data,
     dataType: 'json',
-    contentType: 'application/json; charset=utf-8',
+    contentType: 'text/plain; charset=utf-8',
     success: function(res) {
       _this.locked = true;
       _this.key = res.key;
@@ -170,8 +170,7 @@ haste.extensionMap = {
   lua: 'lua', pas: 'delphi', java: 'java', cpp: 'cpp', cc: 'cpp', m: 'objectivec',
   vala: 'vala', sql: 'sql', sm: 'smalltalk', lisp: 'lisp', ini: 'ini',
   diff: 'diff', bash: 'bash', sh: 'bash', tex: 'tex', erl: 'erlang', hs: 'haskell',
-  md: 'markdown', txt: '', coffee: 'coffee', json: 'javascript',
-  swift: 'swift'
+  md: 'markdown', txt: '', coffee: 'coffee', swift: 'swift'
 };
 
 // Look up the extension preferred for a type

static/index.html

@@ -1,13 +1,12 @@
 <html>
-
 	<head>
-
-		<title>hastebin</title>
+		<title>Webionite hastebin</title>
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
 		<meta charset="utf-8" />
-		<link rel="stylesheet" type="text/css" href="solarized_dark.css"/>
+		<link rel="stylesheet" type="text/css" href="tomorrow.css"/>
 		<link rel="stylesheet" type="text/css" href="application.css"/>
 
-		<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+		<script type="text/javascript" src="jquery.min.js"></script>
 		<script type="text/javascript" src="highlight.min.js"></script>
 		<script type="text/javascript" src="application.min.js"></script>
 
@@ -39,30 +38,27 @@
 	</head>
 
 	<body>
-		<ul id="messages"></ul>
 
 		<div id="key">
-		  <div id="pointer" style="display:none;"></div>
-			<div id="box1">
-				<a href="/about.md" class="logo"></a>
-			</div>
-			<div id="box2">
-				<button class="save function button-picture">Save</button>
-				<button class="new function button-picture">New</button>
-				<button class="duplicate function button-picture">Duplicate & Edit</button>
-				<button class="raw function button-picture">Just Text</button>
-				<button class="twitter function button-picture">Twitter</button>
-			</div>
-			<div id="box3" style="display:none;">
-				<div class="label"></div>
-				<div class="shortcut"></div>
+			<a href="/about.md" class="logo"></a>
+			<div>
+				<div id="pointer" style="display:none;"></div>
+				<div id="box2">
+					<button class="save function button-picture">Save</button>
+					<button class="new function button-picture">New</button>
+					<button class="duplicate function button-picture">Duplicate &amp; Edit</button>
+					<button class="raw function button-picture">Just Text</button>
+				</div>
+				<div id="box3" style="display:none;">
+					<div class="label"></div>
+					<div class="shortcut"></div>
+				</div>
+				<ul id="messages"></ul>
 			</div>
 		</div>
 
 		<div id="linenos"></div>
 		<pre id="box" style="display:none;" class="hljs" tabindex="0"><code></code></pre>
 		<textarea spellcheck="false" style="display:none;"></textarea>
-
 	</body>
-
 </html>

static/solarized_dark.css

@@ -1,84 +0,0 @@
-/*
-
-Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull <sourdrums@gmail.com>
-
-*/
-
-.hljs {
-  display: block;
-  overflow-x: auto;
-  padding: 0.5em;
-  background: #002b36;
-  color: #839496;
-}
-
-.hljs-comment,
-.hljs-quote {
-  color: #586e75;
-}
-
-/* Solarized Green */
-.hljs-keyword,
-.hljs-selector-tag,
-.hljs-addition {
-  color: #859900;
-}
-
-/* Solarized Cyan */
-.hljs-number,
-.hljs-string,
-.hljs-meta .hljs-meta-string,
-.hljs-literal,
-.hljs-doctag,
-.hljs-regexp {
-  color: #2aa198;
-}
-
-/* Solarized Blue */
-.hljs-title,
-.hljs-section,
-.hljs-name,
-.hljs-selector-id,
-.hljs-selector-class {
-  color: #268bd2;
-}
-
-/* Solarized Yellow */
-.hljs-attribute,
-.hljs-attr,
-.hljs-variable,
-.hljs-template-variable,
-.hljs-class .hljs-title,
-.hljs-type {
-  color: #b58900;
-}
-
-/* Solarized Orange */
-.hljs-symbol,
-.hljs-bullet,
-.hljs-subst,
-.hljs-meta,
-.hljs-meta .hljs-keyword,
-.hljs-selector-attr,
-.hljs-selector-pseudo,
-.hljs-link {
-  color: #cb4b16;
-}
-
-/* Solarized Red */
-.hljs-built_in,
-.hljs-deletion {
-  color: #dc322f;
-}
-
-.hljs-formula {
-  background: #073642;
-}
-
-.hljs-emphasis {
-  font-style: italic;
-}
-
-.hljs-strong {
-  font-weight: bold;
-}

static/tomorrow.css

@@ -0,0 +1,72 @@
+/* http://jmblog.github.com/color-themes-for-google-code-highlightjs */
+
+/* Tomorrow Comment */
+.hljs-comment,
+.hljs-quote {
+  color: #8e908c;
+}
+
+/* Tomorrow Red */
+.hljs-variable,
+.hljs-template-variable,
+.hljs-tag,
+.hljs-name,
+.hljs-selector-id,
+.hljs-selector-class,
+.hljs-regexp,
+.hljs-deletion {
+  color: #c82829;
+}
+
+/* Tomorrow Orange */
+.hljs-number,
+.hljs-built_in,
+.hljs-builtin-name,
+.hljs-literal,
+.hljs-type,
+.hljs-params,
+.hljs-meta,
+.hljs-link {
+  color: #f5871f;
+}
+
+/* Tomorrow Yellow */
+.hljs-attribute {
+  color: #eab700;
+}
+
+/* Tomorrow Green */
+.hljs-string,
+.hljs-symbol,
+.hljs-bullet,
+.hljs-addition {
+  color: #718c00;
+}
+
+/* Tomorrow Blue */
+.hljs-title,
+.hljs-section {
+  color: #4271ae;
+}
+
+/* Tomorrow Purple */
+.hljs-keyword,
+.hljs-selector-tag {
+  color: #8959a8;
+}
+
+.hljs {
+  display: block;
+  overflow-x: auto;
+  background: white;
+  color: #4d4d4c;
+  padding: 0.5em;
+}
+
+.hljs-emphasis {
+  font-style: italic;
+}
+
+.hljs-strong {
+  font-weight: bold;
+}