diff --git a/.gitignore b/.gitignore index 589f639..b6ec2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,9 @@ config.json images # The agent CA keys and certificates (custom.pkiDir). Never commit the keys. /pki +# The fog-node log file and its rotated copies (api/lib/file-logger.js). The +# *.log rule above does not match fog-node.log.1 to .5. +/logs ################################################ # Dependencies diff --git a/api/lib/file-logger.js b/api/lib/file-logger.js new file mode 100644 index 0000000..429befe --- /dev/null +++ b/api/lib/file-logger.js @@ -0,0 +1,112 @@ +const fs = require('fs'); +const path = require('path'); +const util = require('util'); + +// The fog-node log file. config/log.js hands create() to captains-log as its +// `custom` logger, so each sails.log.* call writes one line to the console and +// to /fog-node.log. captains-log filters by sails.config.log.level +// before it calls the logger, so this module does no level filtering. +// +// All file work is synchronous: a line never arrives during a rotation, and +// no line is lost or reordered. Node built-ins only, so it runs on any OS. + +const FILE = 'fog-node.log'; +const MAX_BYTES = 10 * 1024 * 1024; +// The number of old files kept: fog-node.log.1 to fog-node.log.5. +const KEEP = 5; +// These levels go to stderr on the console, the rest to stdout. +const STDERR_LEVELS = new Set(['error', 'warn', 'crit']); +const LEVELS = ['log', 'error', 'warn', 'info', 'verbose', 'debug', 'silly', 'crit']; + +// The log folder: sails.config.custom.logDir when it is a non-empty string, +// else fallbackDir. The Log Viewer calls this too, so the rule lives once. +const resolveDir = ({fallbackDir}) => { + const custom = global.sails && global.sails.config && global.sails.config.custom; + const dir = custom && custom.logDir; + return typeof dir === 'string' && dir !== '' ? dir : fallbackDir; +}; + +// logDir and maxBytes are for tests. Without them the folder comes from +// resolveDir at the first write, and maxBytes is MAX_BYTES. +const create = ({fallbackDir, logDir, maxBytes = MAX_BYTES} = {}) => { + let dir; + let fd; + let size = 0; + // 'new' until the first write, 'open' with a file, 'off' after a failure + // (the console only, until the next start). + let state = 'new'; + // After a failed rotation: the size at which to try again. + let retryAt = 0; + + const notice = (message) => process.stderr.write(`${new Date().toISOString()} error: ${message}\n`); + const file = (n) => path.join(dir, n ? `${FILE}.${n}` : FILE); + + const open = () => { + fd = fs.openSync(file(0), 'a'); + size = fs.fstatSync(fd).size; + }; + + const start = () => { + dir = logDir || resolveDir({fallbackDir}); + try { + fs.mkdirSync(dir, {recursive: true}); + open(); + state = 'open'; + } catch (err) { + state = 'off'; + notice(`The log file in ${dir} cannot be opened, so fog-node logs to the console only: ${err.message}`); + } + }; + + // Close the file, shift .4 to .5 ... .1 to .2, the file to .1, and open a + // new file. A rename replaces its target, so the old .5 is gone. + const rotate = () => { + fs.closeSync(fd); + try { + for (let n = KEEP - 1; n >= 0; n--) { + try { + fs.renameSync(file(n), file(n + 1)); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } + } + open(); + retryAt = 0; + } catch (err) { + // On Windows another program can hold the file without share-delete. + notice(`The log file in ${dir} cannot be rotated, so it grows for now: ${err.message}`); + open(); + retryAt = size + maxBytes / 10; + } + }; + + const writeFile = (text) => { + if (state === 'new') start(); + if (state !== 'open') return; + try { + const bytes = Buffer.byteLength(text); + if (size > 0 && size + bytes > maxBytes && size >= retryAt) rotate(); + fs.writeSync(fd, text); + size += bytes; + } catch (err) { + state = 'off'; + notice(`The log file in ${dir} cannot be written, so fog-node logs to the console only: ${err.message}`); + } + }; + + const write = (level, args) => { + const text = (level === 'blank' ? '' : `${new Date().toISOString()} ${level}: ${util.format(...args)}`) + '\n'; + try { + (STDERR_LEVELS.has(level) ? process.stderr : process.stdout).write(text); + } catch { + // A closed pipe must not stop the file log. + } + writeFile(text); + }; + + const logger = {blank: (...args) => write('blank', args)}; + for (const level of LEVELS) logger[level] = (...args) => write(level, args); + return logger; +}; + +module.exports = {resolveDir, create}; diff --git a/config/log.js b/config/log.js index 1ef07f5..9bba1cc 100644 --- a/config/log.js +++ b/config/log.js @@ -10,6 +10,8 @@ * https://sailsjs.com/docs/concepts/logging */ +const path = require('path'); + module.exports.log = { /*************************************************************************** @@ -26,4 +28,10 @@ module.exports.log = { // level: 'info' + // Each line goes to the console and to /fog-node.log + // (api/lib/file-logger.js). The folder is custom.logDir, else /logs. + // inspect is off so that the logger gets the raw arguments. + inspect: false, + custom: require('../api/lib/file-logger').create({fallbackDir: path.resolve(__dirname, '..', 'logs')}) + }; diff --git a/docs/install.md b/docs/install.md index 3059c0e..727b93c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -265,6 +265,18 @@ the fog-node directory, and run `node app.js --prod` (what `npm start` runs). Start fog-node after MongoDB and Redis. If fog-node starts first, it fails to connect and exits, so set the service to restart on failure. +### The log file + +fog-node writes each log line to the console and to the file `fog-node.log`. +By default the file is in the `logs` folder of the fog-node directory. To move +it, set `custom.logDir` in `config/local.js`. A setup re-run keeps that value. + +When `fog-node.log` passes 10 MB, fog-node renames it to `fog-node.log.1` and +starts a new file. It keeps five old files, `fog-node.log.1` to `fog-node.log.5`. + +The production log level is `debug`. At that level `info`, `verbose` and +`silly` lines are not written, because the log library ranks them below `debug`. + ## Update fog-node Run these steps in the fog-node directory, in this order: diff --git a/test/lib/file-logger.test.js b/test/lib/file-logger.test.js new file mode 100644 index 0000000..63518a4 --- /dev/null +++ b/test/lib/file-logger.test.js @@ -0,0 +1,133 @@ +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const fileLogger = require('../../api/lib/file-logger'); + +// The file log (api/lib/file-logger.js): the format, the console split, the +// rotation and the unwritable folder. Each test drives create() with its own +// temporary folder. +describe('File logger::', () => { + const LINE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z (\w+): (.*)$/; + let dir; + let out; + let err; + let realOut; + let realErr; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fog-node-filelog-')); + out = []; + err = []; + realOut = process.stdout.write; + realErr = process.stderr.write; + process.stdout.write = (chunk) => { out.push(String(chunk)); return true; }; + process.stderr.write = (chunk) => { err.push(String(chunk)); return true; }; + }); + + afterEach(() => { + process.stdout.write = realOut; + process.stderr.write = realErr; + fs.rmSync(dir, {recursive: true, force: true}); + }); + + const read = (name) => fs.readFileSync(path.join(dir, name), 'utf8'); + + it('writes the format to the file and to the console', () => { + const logger = fileLogger.create({fallbackDir: dir, logDir: dir}); + logger.warn('audit: could not read', 'x'); + logger.info('hello %s', 'there'); + logger.error('bad'); + logger.blank(); + const lines = read('fog-node.log').split('\n'); + assert.strictEqual(lines.length, 5); + assert.strictEqual(lines[0].match(LINE)[1], 'warn'); + assert.ok(lines[0].endsWith('warn: audit: could not read x')); + assert.ok(lines[1].endsWith('info: hello there')); + assert.ok(lines[2].endsWith('error: bad')); + assert.strictEqual(lines[3], ''); + assert.strictEqual(lines[4], ''); + // The console gets the same line. error and warn go to stderr, info to stdout. + assert.deepStrictEqual(err, [`${lines[0]}\n`, `${lines[2]}\n`]); + assert.deepStrictEqual(out, [`${lines[1]}\n`, '\n']); + }); + + it('sends crit to stderr and the other levels to stdout', () => { + const logger = fileLogger.create({fallbackDir: dir, logDir: dir}); + logger.crit('c'); + for (const level of ['log', 'verbose', 'debug', 'silly']) logger[level]('m'); + assert.strictEqual(err.length, 1); + assert.ok(err[0].includes(' crit: c')); + assert.strictEqual(out.length, 4); + }); + + it('rotates past maxBytes and keeps five old files', () => { + const logger = fileLogger.create({fallbackDir: dir, logDir: dir, maxBytes: 200}); + const text = 'x'.repeat(100); + logger.error(`first ${text}`); + logger.error(`second ${text}`); + assert.ok(read('fog-node.log.1').includes('first')); + assert.ok(read('fog-node.log').includes('second')); + for (let i = 0; i < 10; i++) logger.error(`more ${i} ${text}`); + const names = fs.readdirSync(dir).sort(); + assert.deepStrictEqual(names, ['fog-node.log', 'fog-node.log.1', 'fog-node.log.2', 'fog-node.log.3', 'fog-node.log.4', 'fog-node.log.5']); + assert.ok(!read('fog-node.log.5').includes('first')); + }); + + it('reports a failed rename once, keeps logging, and retries after maxBytes / 10 more bytes', () => { + // A fog-node.log.4 file that cannot replace a non-empty folder named + // fog-node.log.5 makes the first rename fail. + fs.writeFileSync(path.join(dir, 'fog-node.log.4'), 'old'); + fs.mkdirSync(path.join(dir, 'fog-node.log.5')); + fs.writeFileSync(path.join(dir, 'fog-node.log.5', 'keep'), 'x'); + const logger = fileLogger.create({fallbackDir: dir, logDir: dir, maxBytes: 1000}); + const rotations = () => err.filter((l) => l.includes('cannot be rotated')).length; + const size = () => fs.statSync(path.join(dir, 'fog-node.log')).size; + // Short lines (about 45 bytes): fill the file until the write that would + // pass 1000 bytes tries to rotate. + for (let i = 0; i < 40 && rotations() === 0; i++) logger.info(`fill ${i}`); + assert.strictEqual(rotations(), 1); + const grown = size(); + // Less than maxBytes / 10 (100) more bytes: no new attempt, the file grows. + logger.info('one'); + assert.strictEqual(rotations(), 1); + assert.ok(size() > grown); + // After 100 more bytes it tries again. + for (let i = 0; i < 4; i++) logger.info(`more ${i}`); + assert.strictEqual(rotations(), 2); + }); + + it('reports an unwritable folder once and still logs to the console', () => { + const blocker = path.join(dir, 'a-file'); + fs.writeFileSync(blocker, 'x'); + const logger = fileLogger.create({fallbackDir: dir, logDir: path.join(blocker, 'logs')}); + assert.doesNotThrow(() => { + logger.warn('one'); + logger.warn('two'); + logger.info('three'); + }); + const notices = err.filter((l) => l.includes('a-file')); + assert.strictEqual(notices.length, 1); + assert.strictEqual(err.filter((l) => / warn: /.test(l)).length, 2); + assert.strictEqual(out.length, 1); + }); + + describe('resolveDir::', () => { + // The mocha lift sets global.sails: put it back after each test. + let lifted; + beforeEach(() => { lifted = global.sails; }); + afterEach(() => { global.sails = lifted; }); + + it('uses sails.config.custom.logDir when it is a non-empty string', () => { + global.sails = {config: {custom: {logDir: '/somewhere'}}}; + assert.strictEqual(fileLogger.resolveDir({fallbackDir: '/fb'}), '/somewhere'); + }); + + it('falls back without sails or with an empty value', () => { + delete global.sails; + assert.strictEqual(fileLogger.resolveDir({fallbackDir: '/fb'}), '/fb'); + global.sails = {config: {custom: {logDir: ''}}}; + assert.strictEqual(fileLogger.resolveDir({fallbackDir: '/fb'}), '/fb'); + }); + }); +}); diff --git a/test/lib/log-file-lift.test.js b/test/lib/log-file-lift.test.js new file mode 100644 index 0000000..98fb08a --- /dev/null +++ b/test/lib/log-file-lift.test.js @@ -0,0 +1,17 @@ +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +// The mocha lift (test/lifecycle.test.js) points custom.logDir at a temporary +// folder. A sails.log.warn line reaches fog-node.log there. warn, not info: +// the lift runs at debug, and captains-log drops info below that. +describe('Log file in the lift::', () => { + it('holds a sails.log.warn line in the configured folder', () => { + const marker = `log-file-lift-${process.pid}-${Date.now()}`; + sails.log.warn(marker); + const file = path.join(sails.config.custom.logDir, 'fog-node.log'); + const line = fs.readFileSync(file, 'utf8').split('\n').find((l) => l.includes(marker)); + assert.ok(line, 'the marker line is in fog-node.log'); + assert.match(line, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z warn: /); + }); +}); diff --git a/test/lifecycle.test.js b/test/lifecycle.test.js index 959aa19..719f67f 100644 --- a/test/lifecycle.test.js +++ b/test/lifecycle.test.js @@ -1,4 +1,7 @@ const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const sails = require('sails'); const support = require('./support/pki'); const {freePort} = require('./support/port'); @@ -13,6 +16,8 @@ let target; // for a free port (test/support/port.js). let pkiDir; let agentTlsPort; +// The log folder of the lift (api/lib/file-logger.js), outside the repo. +const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fog-node-test-log-')); // The shared PKI fixture is built before the lift, so the lift-time // ensureServerCert (with the same SAN list) changes nothing in the read-only @@ -36,7 +41,8 @@ before(async function() { datastores: {[target.datastore]: target.overrides}, // config/bootstrap.js refuses to lift on any other database than // testDatabase, before its index writes. - custom: {pkiDir, agentTlsPort, agentTlsSans: support.SANS, testDatabase: target.database} + // logDir: a temporary folder, so the suite never writes into the repo. + custom: {pkiDir, agentTlsPort, agentTlsSans: support.SANS, testDatabase: target.database, logDir} }, (err) => (err ? reject(err) : resolve())); }); // The guard, before any test file runs: the database the models really @@ -64,6 +70,8 @@ after(async () => { // The seam writes after the response finishes, so let the last write land. await new Promise((resolve) => setTimeout(resolve, 200)); await new Promise((resolve, reject) => sails.lower((err) => (err ? reject(err) : resolve()))); + // The lift's log folder (a temporary folder, see logDir above). + fs.rmSync(logDir, {recursive: true, force: true}); }); describe('Basic Tests ::', () => { diff --git a/test/setup/read-old-local.test.js b/test/setup/read-old-local.test.js new file mode 100644 index 0000000..5d0185f --- /dev/null +++ b/test/setup/read-old-local.test.js @@ -0,0 +1,38 @@ +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const {readOldLocal, customLines} = require('../../tools/setup/pki'); + +// What the setup tools keep from an existing config/local.js +// (tools/setup/pki.js readOldLocal and customLines). +describe('Setup keeps custom.logDir::', () => { + let app; + + beforeEach(() => { + app = fs.mkdtempSync(path.join(os.tmpdir(), 'fog-node-oldlocal-')); + fs.mkdirSync(path.join(app, 'config')); + }); + + afterEach(() => fs.rmSync(app, {recursive: true, force: true})); + + it('readOldLocal returns the logDir of an existing local.js', () => { + fs.writeFileSync(path.join(app, 'config', 'local.js'), "module.exports = {custom: {logDir: '/var/log/fog-node', pkiDir: '/p'}};\n"); + const old = readOldLocal(app); + assert.strictEqual(old.logDir, '/var/log/fog-node'); + assert.strictEqual(old.pkiDir, '/p'); + }); + + it('readOldLocal returns no logDir without the file or the setting', () => { + assert.strictEqual(readOldLocal(app).logDir, undefined); + fs.writeFileSync(path.join(app, 'config', 'local.js'), 'module.exports = {custom: {}};\n'); + assert.strictEqual(readOldLocal(app).logDir, undefined); + }); + + it('customLines writes a logDir line only when there is one', () => { + assert.ok(customLines({logDir: '/x'}).includes("logDir: '/x'")); + assert.ok(!customLines({}).includes('logDir')); + assert.ok(customLines({pkiOverride: '/p', logDir: '/x'}).includes("pkiDir: '/p'")); + assert.strictEqual(customLines({}), ''); + }); +}); diff --git a/tools/setup/index.js b/tools/setup/index.js index f7d4c31..ebfbe8b 100644 --- a/tools/setup/index.js +++ b/tools/setup/index.js @@ -25,9 +25,8 @@ const welcome = `Welcome to the FOG installer.\nYou will be guided through confi var COMPLETED = false; var payload = {}; -// Escape a value for a single-quoted JS string in a generated config file. -// A Windows path has backslashes, which would otherwise become escapes. -const jsStr = (s) => String(s).replace(/\\/g, '\\\\').replace(/'/g, '\\\''); +// The single-quoted JS string escaper for the generated config (tools/setup/pki.js). +const {jsStr} = pki; // The agent CA backup passphrase (tools/setup/pki.js setupPki). purpose is // 'new', 'replace' or 'restore'. Returns a normalized passphrase, or null. @@ -215,8 +214,8 @@ async.waterfall([ // The server certificate of the agent listener names this computer and // the host of each URL that the installer was given. payload.agentTlsSans = cleanSans([...defaultSans(), ...urlHosts(splitList(payload.server.urls))]); - const pkiLines = pkiOverride ? `, - pkiDir: '${jsStr(pkiOverride)}'` : ''; + // A kept custom.logDir goes next to it. + const pkiLines = pki.customLines({pkiOverride, logDir: oldLocal.logDir}); // The kept http.trustProxy: behind a reverse proxy it makes the audit rows // record the client address. JSON is valid JS for a boolean, a number, a // string or a list. diff --git a/tools/setup/pki.js b/tools/setup/pki.js index 3d34477..2e255f5 100644 --- a/tools/setup/pki.js +++ b/tools/setup/pki.js @@ -196,13 +196,13 @@ const setupPki = async ({pkiDir, newRoot, settings, log, command, sans, escrows, /** * What the setup tools keep from an existing config/local.js: custom.pkiDir - * (the CA keys are there) and http.trustProxy. Both are undefined without the - * file. The setup tools carry them into the local.js they write, so a re-run - * keeps them. + * (the CA keys are there), custom.logDir and http.trustProxy. All are + * undefined without the file. The setup tools carry them into the local.js + * they write, so a re-run keeps them. */ const readOldLocal = (appPath) => { const localCfg = path.join(appPath, 'config', 'local.js'); - if (!exists(localCfg)) return {pkiDir: undefined, trustProxy: undefined}; + if (!exists(localCfg)) return {pkiDir: undefined, logDir: undefined, trustProxy: undefined}; const local = require(localCfg) || {}; // The installer writes a new local.js after this read. Left in the require // cache, the old file is what its later steps (schema, Sails config, @@ -210,10 +210,22 @@ const readOldLocal = (appPath) => { delete require.cache[require.resolve(localCfg)]; return { pkiDir: local.custom ? local.custom.pkiDir : undefined, + logDir: local.custom ? local.custom.logDir : undefined, trustProxy: local.http ? local.http.trustProxy : undefined }; }; +// Escape a value for a single-quoted JS string in a generated config file. +// A Windows path has backslashes, which would otherwise become escapes. +const jsStr = (s) => String(s).replace(/\\/g, '\\\\').replace(/'/g, '\\\''); + +// The custom: {...} override lines of the local.js that setup writes: a kept +// pkiDir and a kept logDir, each only when there is one. Each line starts with +// a comma, so the text follows the last fixed key. +const customLines = ({pkiOverride, logDir}) => (pkiOverride ? `, + pkiDir: '${jsStr(pkiOverride)}'` : '') + (logDir ? `, + logDir: '${jsStr(logDir)}'` : ''); + // The default PKI folder, /pki, as config/custom.js. const defaultPkiDir = (appPath) => path.join(appPath, 'pki'); // The folder the setup tools use: the local.js override, else the default. @@ -237,7 +249,7 @@ const printPassphrase = (words) => { const readPassphraseFile = (file) => escrow.normalize(fs.readFileSync(file, 'utf8').split(/\r?\n/)[0]); module.exports = { - setupPki, isRootError, readOldLocal, defaultPkiDir, pkiDirFor, printPassphrase, readPassphraseFile, INTERACTIVE_TRIES, + setupPki, isRootError, readOldLocal, customLines, jsStr, defaultPkiDir, pkiDirFor, printPassphrase, readPassphraseFile, INTERACTIVE_TRIES, // For tools/setup/agent-ca-lib.js. saveBackup, restoreBackup, findEscrow, settingValue, writeSetting, fingerprintIn }; diff --git a/tools/smoke/install.js b/tools/smoke/install.js index fd9cf75..74ecfcd 100644 --- a/tools/smoke/install.js +++ b/tools/smoke/install.js @@ -185,7 +185,8 @@ async function main() { fs.writeFileSync(configs[0], `module.exports = {custom: {pkiDir: ${JSON.stringify(pkiDir)}}};\n`); writeAnswers(); await step('the installer, from an answers file', runInstaller); - child = server.start({port, prod: true, env: {}, log}); + // The log file goes into the run's work folder, not into the checkout. + child = server.start({port, prod: true, env: {'sails_custom__logDir': path.join(work, 'logs')}, log}); await step('production lift with the installed config', () => server.waitForReady(child, base)); await step('sign in through the login form', signIn); await step('sockets accept the installed URL', async () => { diff --git a/tools/smoke/ui.js b/tools/smoke/ui.js index 26a8948..436545a 100644 --- a/tools/smoke/ui.js +++ b/tools/smoke/ui.js @@ -46,6 +46,7 @@ */ const crypto = require('crypto'); const fs = require('fs'); +const os = require('os'); const path = require('path'); const {chromium} = require('playwright'); const server = require('./lib/server'); @@ -61,6 +62,9 @@ const password = crypto.randomBytes(12).toString('hex'); // bundles (issue #115 broke that build, and nothing noticed). const prod = process.argv.includes('--prod'); const outDir = path.join(root, 'ui-smoke-out'); +// The server log file (api/lib/file-logger.js) goes to a temporary folder, so +// the run never writes a logs/ folder into the checkout. +const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fog-ui-smoke-log-')); // Every page in config/routes.js that renders the layout, except the edit // pages, which need a record id (checkRoundTrip opens them). The sidebar hides @@ -249,6 +253,7 @@ async function main() { env: { 'sails_sockets__onlyAllowOrigins': JSON.stringify([base]), 'sails_custom__testDatabase': target.database, + 'sails_custom__logDir': logDir, ...datastoreEnv } }); @@ -319,6 +324,7 @@ async function main() { } finally { if (browser) await browser.close(); server.stop(child); + fs.rmSync(logDir, {recursive: true, force: true}); } if (failed) { console.log(`\n${failed} check(s) failed. Last fog-node output:\n${serverLog.slice(-40).join('\n')}`);