Add official MySQL support. Simplify database options.

package-lock.json

@@ -115,6 +115,11 @@
         "color-convert": "^1.9.0"
       }
     },
+    "ansicolors": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.3.2.tgz",
+      "integrity": "sha1-ZlWX3oap/+Oqm/vmyuXG6kJrSXk="
+    },
     "aproba": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -382,6 +387,15 @@
       "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
       "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
     },
+    "cardinal": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-2.1.1.tgz",
+      "integrity": "sha1-fMEFXYItISlU0HsIXeolHMe8VQU=",
+      "requires": {
+        "ansicolors": "~0.3.2",
+        "redeyed": "~2.1.0"
+      }
+    },
     "caseless": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
@@ -799,6 +813,11 @@
       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
       "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
     },
+    "denque": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
+      "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
+    },
     "detect-file": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
@@ -1063,8 +1082,7 @@
     "esprima": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
-      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
-      "dev": true
     },
     "esquery": {
       "version": "1.3.1",
@@ -1482,6 +1500,14 @@
         }
       }
     },
+    "generate-function": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
+      "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
+      "requires": {
+        "is-property": "^1.0.2"
+      }
+    },
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -1985,6 +2011,11 @@
         "isobject": "^3.0.1"
       }
     },
+    "is-property": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
+      "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ="
+    },
     "is-regexp": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz",
@@ -2587,11 +2618,24 @@
         }
       }
     },
+    "long": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+      "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
+    },
     "lowercase-keys": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
       "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA=="
     },
+    "lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "requires": {
+        "yallist": "^3.0.2"
+      }
+    },
     "make-iterator": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/make-iterator/-/make-iterator-1.0.1.tgz",
@@ -2769,6 +2813,56 @@
         }
       }
     },
+    "mysql2": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.1.0.tgz",
+      "integrity": "sha512-9kGVyi930rG2KaHrz3sHwtc6K+GY9d8wWk1XRSYxQiunvGcn4DwuZxOwmK11ftuhhwrYDwGx9Ta4VBwznJn36A==",
+      "requires": {
+        "cardinal": "^2.1.1",
+        "denque": "^1.4.1",
+        "generate-function": "^2.3.1",
+        "iconv-lite": "^0.5.0",
+        "long": "^4.0.0",
+        "lru-cache": "^5.1.1",
+        "named-placeholders": "^1.1.2",
+        "seq-queue": "^0.0.5",
+        "sqlstring": "^2.3.1"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.5.2",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz",
+          "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3"
+          }
+        }
+      }
+    },
+    "named-placeholders": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz",
+      "integrity": "sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==",
+      "requires": {
+        "lru-cache": "^4.1.3"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "4.1.5",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+          "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+          "requires": {
+            "pseudomap": "^1.0.2",
+            "yallist": "^2.1.2"
+          }
+        },
+        "yallist": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+          "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
+        }
+      }
+    },
     "nanomatch": {
       "version": "1.2.13",
       "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@@ -3276,6 +3370,11 @@
       "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
       "dev": true
     },
+    "pseudomap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
+    },
     "psl": {
       "version": "1.8.0",
       "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
@@ -3345,6 +3444,14 @@
         "resolve": "^1.1.6"
       }
     },
+    "redeyed": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-2.1.1.tgz",
+      "integrity": "sha1-iYS1gV2ZyyIEacme7v/jiRPmzAs=",
+      "requires": {
+        "esprima": "~4.0.0"
+      }
+    },
     "regex-not": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
@@ -3516,6 +3623,11 @@
       "integrity": "sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw==",
       "dev": true
     },
+    "seq-queue": {
+      "version": "0.0.5",
+      "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
+      "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4="
+    },
     "set-blocking": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
@@ -3743,6 +3855,11 @@
         "node-pre-gyp": "^0.11.0"
       }
     },
+    "sqlstring": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz",
+      "integrity": "sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg=="
+    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",

package.json

@@ -28,6 +28,7 @@
     "mime": "^2.4.4",
     "moment": "^2.24.0",
     "mv": "^2.1.1",
+    "mysql2": "^2.1.0",
     "public-ip": "^4.0.0",
     "sqlite3": "^5.0.0",
     "tmp": "^0.1.0",

src/cfg.js

@@ -102,22 +102,15 @@ for (const [key, value] of Object.entries(config)) {
   delete config[key];
 }
 
-if (! config["knex"]) {
+if (! config.dbType) {
-  config.knex = {
+  config.dbType = "sqlite";
-    client: "sqlite",
-    connection: {
-      filename: path.join(config.dbDir, "data.sqlite")
-    },
-    useNullAsDefault: true
-  };
 }
 
-// Make sure migration settings are always present in knex config
+if (! config.sqliteOptions) {
-Object.assign(config["knex"], {
+  config.sqliteOptions = {
-  migrations: {
+    filename: path.resolve(__dirname, "..", "db", "data.sqlite"),
-    directory: path.join(config.dbDir, "migrations")
+  };
 }
-});
 
 // Move greetingMessage/greetingAttachment to the guildGreetings object internally
 // Or, in other words, if greetingMessage and/or greetingAttachment is set, it is applied for all servers that don't
@@ -150,8 +143,8 @@ for (const [key, value] of Object.entries(config)) {
 const ajv = new Ajv({ useDefaults: true, coerceTypes: "array" });
 
 // https://github.com/ajv-validator/ajv/issues/141#issuecomment-270692820
-const truthyValues = ["1", "true", "on"];
+const truthyValues = ["1", "true", "on", "yes"];
-const falsyValues = ["0", "false", "off"];
+const falsyValues = ["0", "false", "off", "no"];
 ajv.addKeyword("coerceBoolean", {
   compile(value) {
     return (data, dataPath, parentData, parentKey) => {
@@ -208,12 +201,18 @@ const validate = ajv.compile(schema);
 const configIsValid = validate(config);
 
 if (! configIsValid) {
-  console.error("Issues with configuration options:");
+  console.error("");
+  console.error("NOTE! Issues with configuration:");
   for (const error of validate.errors) {
-    console.error(`The "${error.dataPath.slice(1)}" option ${error.message}`);
+    if (error.params.missingProperty) {
+      console.error(`- Missing required option: "${error.params.missingProperty.slice(1)}"`);
+    } else {
+      console.error(`- The "${error.dataPath.slice(1)}" option ${error.message}`);
+    }
   }
   console.error("");
   console.error("Please restart the bot after fixing the issues mentioned above.");
+  console.error("");
   process.exit(1);
 }
 

src/data/cfg.jsdoc.js

@@ -15,7 +15,7 @@
  * @property {string} [mentionRole="here"]
  * @property {boolean} [pingOnBotMention=true]
  * @property {string} [botMentionResponse]
- * @property {array} [inboxServerPermission]
+ * @property {array} [inboxServerPermission=[]]
  * @property {boolean} [alwaysReply=false]
  * @property {boolean} [alwaysReplyAnon=false]
  * @property {boolean} [useNicknames=false]
@@ -39,7 +39,7 @@
  * @property {string} [timeOnServerDeniedMessage="You haven't been a member of the server for long enough to contact modmail."]
  * @property {boolean} [relaySmallAttachmentsAsAttachments=false]
  * @property {number} [smallAttachmentLimit=2097152] Max size of attachment to relay directly. Default is 2MB.
- * @property {string} [attachmentStorage="local"]
+ * @property {string} [attachmentStorage="original"]
  * @property {string} [attachmentStorageChannelId]
  * @property {*} [categoryAutomation={}]
  * @property {boolean} [updateNotifications=true]
@@ -51,4 +51,7 @@
  * @property {object} [knex]
  * @property {string} [logDir]
  * @property {array} [extraIntents=[]]
+ * @property {*} [dbType="sqlite"]
+ * @property {*} [sqliteOptions]
+ * @property {*} [mysqlOptions]
  */

src/data/cfg.schema.json

@@ -100,7 +100,8 @@
     },
 
     "inboxServerPermission": {
-      "$ref": "#/definitions/stringArray"
+      "$ref": "#/definitions/stringArray",
+      "default": []
     },
     "alwaysReply": {
       "$ref": "#/definitions/customBoolean",
@@ -253,26 +254,72 @@
     },
 
     "dbDir": {
-      "type": "string"
+      "type": "string",
+      "deprecationMessage": "This option is deprecated. Please use sqliteOptions instead."
     },
+
     "knex": {
-      "type": "object"
+      "type": "object",
+      "deprecationMessage": "This option is deprecated. Please use dbType and sqliteOptions/mysqlOptions instead."
     },
 
     "logDir": {
       "type": "string",
-      "deprecationMessage": "This option is no longer used"
+      "deprecationMessage": "This option is deprecated. Logs are no longer stored in individual files."
     },
 
     "extraIntents": {
       "$ref": "#/definitions/stringArray",
       "default": []
+    },
+
+    "dbType": {
+      "anyOf": [
+        { "const": "sqlite" },
+        { "const": "mysql" }
+      ],
+      "default": "sqlite"
+    },
+
+    "sqliteOptions": {
+      "type": "object",
+      "properties": {
+        "filename": {
+          "type": "string"
+        }
+      },
+      "required": ["filename"]
+    },
+
+    "mysqlOptions": {
+      "type": "object",
+      "properties": {
+        "host": {
+          "type": "string"
+        },
+        "port": {
+          "type": "number"
+        },
+        "user": {
+          "type": "string"
+        },
+        "password": {
+          "type": "string"
+        },
+        "database": {
+          "type": "string"
+        },
+        "timezone": {
+          "type": "string"
+        }
+      },
+      "required": ["user", "password", "database"]
     }
   },
   "allOf": [
     {
       "$comment": "Base required values",
-      "required": ["token", "mainGuildId", "mailGuildId", "logChannelId"]
+      "required": ["token", "mainGuildId", "mailGuildId", "logChannelId", "dbType"]
     },
     {
       "$comment": "Make attachmentStorageChannelId required if attachmentStorage is set to 'discord'",
@@ -287,6 +334,32 @@
       "then": {
         "required": ["attachmentStorageChannelId"]
       }
+    },
+    {
+      "$comment": "Make sqliteOptions required if dbType is set to 'sqlite'",
+      "if": {
+        "properties": {
+          "dbType": {
+            "const": "sqlite"
+          }
+        }
+      },
+      "then": {
+        "required": ["sqliteOptions"]
+      }
+    },
+    {
+      "$comment": "Make mysqlOptions required if dbType is set to 'mysql'",
+      "if": {
+        "properties": {
+          "dbType": {
+            "const": "mysql"
+          }
+        }
+      },
+      "then": {
+        "required": ["mysqlOptions"]
+      }
     }
   ]
 }

src/knex.js

@@ -1,6 +1,38 @@
+const path = require("path");
 const config = require("./cfg");
+
+let knexOptions;
+if (config.dbType === "sqlite") {
+  const resolvedPath = path.resolve(process.cwd(), config.sqliteOptions.filename);
+  console.log(`Using an SQLite database:\n  ${resolvedPath}`);
+
+  knexOptions = {
+    client: "sqlite",
+    connection: {
+      ...config.sqliteOptions,
+    },
+  };
+} else if (config.dbType === "mysql") {
+  const host = config.mysqlOptions.host || "localhost";
+  const port = config.mysqlOptions.port || 3306;
+  const mysqlStr = `${config.mysqlOptions.user}@${host}:${port}/${config.mysqlOptions.database}`;
+  console.log(`Using a MySQL database:\n  ${mysqlStr}`);
+
+  knexOptions = {
+    client: "mysql2",
+    connection: {
+      ...config.mysqlOptions,
+    },
+  };
+}
+
 module.exports = require("knex")({
-  ...config.knex,
+  ...knexOptions,
+
+  useNullAsDefault: true,
+  migrations: {
+    directory: path.resolve(__dirname, "..", "db", "migrations"),
+  },
   log: {
     warn(message) {
       if (message.startsWith("FS-related option specified for migration configuration")) {