From ab6b84e6ded4a313267bd91c5c6f24da081efbfd Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Thu, 13 Aug 2020 03:31:48 +0300 Subject: [PATCH] Add official MySQL support. Simplify database options. --- package-lock.json | 121 ++++++++++++++++++++++++++++++++++++++- package.json | 1 + src/cfg.js | 35 ++++++----- src/data/cfg.jsdoc.js | 7 ++- src/data/cfg.schema.json | 83 +++++++++++++++++++++++++-- src/knex.js | 34 ++++++++++- 6 files changed, 253 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3dd0ee9..876df54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -115,6 +115,11 @@ "color-convert": "^1.9.0" } }, + "ansicolors": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.3.2.tgz", + "integrity": "sha1-ZlWX3oap/+Oqm/vmyuXG6kJrSXk=" + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -382,6 +387,15 @@ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, + "cardinal": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-2.1.1.tgz", + "integrity": "sha1-fMEFXYItISlU0HsIXeolHMe8VQU=", + "requires": { + "ansicolors": "~0.3.2", + "redeyed": "~2.1.0" + } + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -799,6 +813,11 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "detect-file": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", @@ -1063,8 +1082,7 @@ "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, "esquery": { "version": "1.3.1", @@ -1482,6 +1500,14 @@ } } }, + "generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "requires": { + "is-property": "^1.0.2" + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1985,6 +2011,11 @@ "isobject": "^3.0.1" } }, + "is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + }, "is-regexp": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", @@ -2587,11 +2618,24 @@ } } }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "requires": { + "yallist": "^3.0.2" + } + }, "make-iterator": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/make-iterator/-/make-iterator-1.0.1.tgz", @@ -2769,6 +2813,56 @@ } } }, + "mysql2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.1.0.tgz", + "integrity": "sha512-9kGVyi930rG2KaHrz3sHwtc6K+GY9d8wWk1XRSYxQiunvGcn4DwuZxOwmK11ftuhhwrYDwGx9Ta4VBwznJn36A==", + "requires": { + "cardinal": "^2.1.1", + "denque": "^1.4.1", + "generate-function": "^2.3.1", + "iconv-lite": "^0.5.0", + "long": "^4.0.0", + "lru-cache": "^5.1.1", + "named-placeholders": "^1.1.2", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.1" + }, + "dependencies": { + "iconv-lite": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", + "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } + } + }, + "named-placeholders": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz", + "integrity": "sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==", + "requires": { + "lru-cache": "^4.1.3" + }, + "dependencies": { + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + } + } + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -3276,6 +3370,11 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" + }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -3345,6 +3444,14 @@ "resolve": "^1.1.6" } }, + "redeyed": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-2.1.1.tgz", + "integrity": "sha1-iYS1gV2ZyyIEacme7v/jiRPmzAs=", + "requires": { + "esprima": "~4.0.0" + } + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", @@ -3516,6 +3623,11 @@ "integrity": "sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw==", "dev": true }, + "seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -3743,6 +3855,11 @@ "node-pre-gyp": "^0.11.0" } }, + "sqlstring": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz", + "integrity": "sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg==" + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", diff --git a/package.json b/package.json index f0f9f0f..23ca0b2 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "mime": "^2.4.4", "moment": "^2.24.0", "mv": "^2.1.1", + "mysql2": "^2.1.0", "public-ip": "^4.0.0", "sqlite3": "^5.0.0", "tmp": "^0.1.0", diff --git a/src/cfg.js b/src/cfg.js index 8364346..35c2371 100644 --- a/src/cfg.js +++ b/src/cfg.js @@ -102,22 +102,15 @@ for (const [key, value] of Object.entries(config)) { delete config[key]; } -if (! config["knex"]) { - config.knex = { - client: "sqlite", - connection: { - filename: path.join(config.dbDir, "data.sqlite") - }, - useNullAsDefault: true - }; +if (! config.dbType) { + config.dbType = "sqlite"; } -// Make sure migration settings are always present in knex config -Object.assign(config["knex"], { - migrations: { - directory: path.join(config.dbDir, "migrations") - } -}); +if (! config.sqliteOptions) { + config.sqliteOptions = { + filename: path.resolve(__dirname, "..", "db", "data.sqlite"), + }; +} // Move greetingMessage/greetingAttachment to the guildGreetings object internally // Or, in other words, if greetingMessage and/or greetingAttachment is set, it is applied for all servers that don't @@ -150,8 +143,8 @@ for (const [key, value] of Object.entries(config)) { const ajv = new Ajv({ useDefaults: true, coerceTypes: "array" }); // https://github.com/ajv-validator/ajv/issues/141#issuecomment-270692820 -const truthyValues = ["1", "true", "on"]; -const falsyValues = ["0", "false", "off"]; +const truthyValues = ["1", "true", "on", "yes"]; +const falsyValues = ["0", "false", "off", "no"]; ajv.addKeyword("coerceBoolean", { compile(value) { return (data, dataPath, parentData, parentKey) => { @@ -208,12 +201,18 @@ const validate = ajv.compile(schema); const configIsValid = validate(config); if (! configIsValid) { - console.error("Issues with configuration options:"); + console.error(""); + console.error("NOTE! Issues with configuration:"); for (const error of validate.errors) { - console.error(`The "${error.dataPath.slice(1)}" option ${error.message}`); + if (error.params.missingProperty) { + console.error(`- Missing required option: "${error.params.missingProperty.slice(1)}"`); + } else { + console.error(`- The "${error.dataPath.slice(1)}" option ${error.message}`); + } } console.error(""); console.error("Please restart the bot after fixing the issues mentioned above."); + console.error(""); process.exit(1); } diff --git a/src/data/cfg.jsdoc.js b/src/data/cfg.jsdoc.js index 339b8f8..26eb805 100644 --- a/src/data/cfg.jsdoc.js +++ b/src/data/cfg.jsdoc.js @@ -15,7 +15,7 @@ * @property {string} [mentionRole="here"] * @property {boolean} [pingOnBotMention=true] * @property {string} [botMentionResponse] - * @property {array} [inboxServerPermission] + * @property {array} [inboxServerPermission=[]] * @property {boolean} [alwaysReply=false] * @property {boolean} [alwaysReplyAnon=false] * @property {boolean} [useNicknames=false] @@ -39,7 +39,7 @@ * @property {string} [timeOnServerDeniedMessage="You haven't been a member of the server for long enough to contact modmail."] * @property {boolean} [relaySmallAttachmentsAsAttachments=false] * @property {number} [smallAttachmentLimit=2097152] Max size of attachment to relay directly. Default is 2MB. - * @property {string} [attachmentStorage="local"] + * @property {string} [attachmentStorage="original"] * @property {string} [attachmentStorageChannelId] * @property {*} [categoryAutomation={}] * @property {boolean} [updateNotifications=true] @@ -51,4 +51,7 @@ * @property {object} [knex] * @property {string} [logDir] * @property {array} [extraIntents=[]] + * @property {*} [dbType="sqlite"] + * @property {*} [sqliteOptions] + * @property {*} [mysqlOptions] */ diff --git a/src/data/cfg.schema.json b/src/data/cfg.schema.json index e03c913..186b8c7 100644 --- a/src/data/cfg.schema.json +++ b/src/data/cfg.schema.json @@ -100,7 +100,8 @@ }, "inboxServerPermission": { - "$ref": "#/definitions/stringArray" + "$ref": "#/definitions/stringArray", + "default": [] }, "alwaysReply": { "$ref": "#/definitions/customBoolean", @@ -253,26 +254,72 @@ }, "dbDir": { - "type": "string" + "type": "string", + "deprecationMessage": "This option is deprecated. Please use sqliteOptions instead." }, + "knex": { - "type": "object" + "type": "object", + "deprecationMessage": "This option is deprecated. Please use dbType and sqliteOptions/mysqlOptions instead." }, "logDir": { "type": "string", - "deprecationMessage": "This option is no longer used" + "deprecationMessage": "This option is deprecated. Logs are no longer stored in individual files." }, "extraIntents": { "$ref": "#/definitions/stringArray", "default": [] + }, + + "dbType": { + "anyOf": [ + { "const": "sqlite" }, + { "const": "mysql" } + ], + "default": "sqlite" + }, + + "sqliteOptions": { + "type": "object", + "properties": { + "filename": { + "type": "string" + } + }, + "required": ["filename"] + }, + + "mysqlOptions": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "database": { + "type": "string" + }, + "timezone": { + "type": "string" + } + }, + "required": ["user", "password", "database"] } }, "allOf": [ { "$comment": "Base required values", - "required": ["token", "mainGuildId", "mailGuildId", "logChannelId"] + "required": ["token", "mainGuildId", "mailGuildId", "logChannelId", "dbType"] }, { "$comment": "Make attachmentStorageChannelId required if attachmentStorage is set to 'discord'", @@ -287,6 +334,32 @@ "then": { "required": ["attachmentStorageChannelId"] } + }, + { + "$comment": "Make sqliteOptions required if dbType is set to 'sqlite'", + "if": { + "properties": { + "dbType": { + "const": "sqlite" + } + } + }, + "then": { + "required": ["sqliteOptions"] + } + }, + { + "$comment": "Make mysqlOptions required if dbType is set to 'mysql'", + "if": { + "properties": { + "dbType": { + "const": "mysql" + } + } + }, + "then": { + "required": ["mysqlOptions"] + } } ] } diff --git a/src/knex.js b/src/knex.js index ff1a9cd..45aa9c5 100644 --- a/src/knex.js +++ b/src/knex.js @@ -1,6 +1,38 @@ +const path = require("path"); const config = require("./cfg"); + +let knexOptions; +if (config.dbType === "sqlite") { + const resolvedPath = path.resolve(process.cwd(), config.sqliteOptions.filename); + console.log(`Using an SQLite database:\n ${resolvedPath}`); + + knexOptions = { + client: "sqlite", + connection: { + ...config.sqliteOptions, + }, + }; +} else if (config.dbType === "mysql") { + const host = config.mysqlOptions.host || "localhost"; + const port = config.mysqlOptions.port || 3306; + const mysqlStr = `${config.mysqlOptions.user}@${host}:${port}/${config.mysqlOptions.database}`; + console.log(`Using a MySQL database:\n ${mysqlStr}`); + + knexOptions = { + client: "mysql2", + connection: { + ...config.mysqlOptions, + }, + }; +} + module.exports = require("knex")({ - ...config.knex, + ...knexOptions, + + useNullAsDefault: true, + migrations: { + directory: path.resolve(__dirname, "..", "db", "migrations"), + }, log: { warn(message) { if (message.startsWith("FS-related option specified for migration configuration")) {