From 0e756ad4d71c6a5d86bd99811b0dcb270f57cbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gera=20G=2E=20G=C3=BCiles?= Date: Mon, 19 Oct 2026 10:23:33 +0200 Subject: [PATCH 1/5] Fixes found while documenting the framework in Forge: CRON step, setTimeout2 limit, AJAX with XML, F.python and F.pipinstall promises, LOGMAIL with an object, IMPORT with an error status, tunnel message types, $.href, gridfields, pending requests in one pass, LDAP bind status, NEWACTION cache option, compact json and ip types --- builders.js | 17 ++++- changelog.txt | 13 ++++ controller.js | 4 +- cron.js | 8 ++- global.js | 4 +- index.js | 65 ++++++++++------- ldap.js | 5 +- proxy.js | 4 +- querybuilder.js | 2 +- tests/common/fixes.js | 158 ++++++++++++++++++++++++++++++++++++++++++ tests/run.js | 2 +- utils.js | 4 +- 12 files changed, 249 insertions(+), 37 deletions(-) create mode 100644 tests/common/fixes.js diff --git a/builders.js b/builders.js index 679b48a..9dd7c03 100755 --- a/builders.js +++ b/builders.js @@ -1428,6 +1428,7 @@ ActionCaller.prototype.exec = function() { let $ = self.$; $.$end = false; + $.$cached = false; $.name = action.name; $.id = action.id; $.error = self.error; @@ -1465,7 +1466,7 @@ ActionCaller.prototype.exec = function() { $.$timeout = null; } - if (!err) { + if (!err && !$.$cached) { if (action.jsoutput) response = action.jsoutput.transform(response).response || {}; @@ -1476,6 +1477,10 @@ ActionCaller.prototype.exec = function() { $.payload[action.extend] = response; response = $.payload; } + + // The "cache" option of the action: the final response is kept for the next call with the same key + if (action.cache && response !== undefined) + action.cache($, response); } if (err) { @@ -1564,6 +1569,16 @@ ActionCaller.prototype.exec = function() { if (self.options.progress) $.progress2 = self.options.progress; + // The "cache" option of the action: a cached response skips the middleware and the action + if (action.cache) { + let cached = action.cache($); + if (cached !== undefined) { + $.$cached = true; + $.$callback(null, cached && typeof(cached) === 'object' ? CLONE(cached) : cached); + return; + } + } + if (action.timeout) { $.$timeout = setTimeout(function() { $.invalid('Timeout.'); diff --git a/changelog.txt b/changelog.txt index c2fd527..14a42fa 100644 --- a/changelog.txt +++ b/changelog.txt @@ -12,6 +12,19 @@ - fixed `AIPARSER()` with tool calling - added a new controller method `$.content(value, content_type)` - added a new event `reset` in the FlowStream +- fixed `CRON()` with the step form (`*/15`, `5/20`): it matches the values of the step and not every value +- fixed the `limit` argument of `setTimeout2()` +- fixed `AJAX()` for the answers with `application/xml` and `text/xml` +- fixed `F.python()` without a callback (the promise resolves with the output) and `F.pipinstall()` without a callback +- fixed `LOGMAIL()` with an object as the body +- fixed `IMPORT()`: an answer with the status 400 or more, or a file that is not a module, gives an error +- fixed the tunnel client (`proxy.js`): the types of the messages "error" and "abort" +- fixed the controller method `$.href()` +- fixed `QueryBuilder.gridfields()`: the accepted fields are applied +- fixed the handling of the pending requests (a single pass instead of `splice()` for each finished request) +- fixed `LDAP()`: a bind with a status other than 0 gives an error even when the server sends no text +- added support for the `cache` option of `NEWACTION()` (the response is kept for the same `query`, `params`, `input`, `user` or `language`) +- added the compact schema types `json` and `ip` (the values are validated) ======================== 0.0.19 diff --git a/controller.js b/controller.js index 095ba1b..0d82625 100755 --- a/controller.js +++ b/controller.js @@ -1081,8 +1081,8 @@ Controller.prototype.$route = function() { }; -Controller.prototype.href = function(key, value) { - return F.TViewEngine.prototype.href.call(this, key, value); +Controller.prototype.href = function() { + return F.TViewEngine.View.prototype.href.apply(this, arguments); }; function readfile(filename, callback) { diff --git a/cron.js b/cron.js index 1da3c3a..c92d720 100644 --- a/cron.js +++ b/cron.js @@ -7,6 +7,9 @@ const Days = ['su', 'mo', 'tu', 'we', 'th', 'fr', 'sa']; // The method compares date with parsed cron data +// The first value of each field: minute, hour, day of the month, month, day of the week +const CRON_MIN = [0, 0, 1, 1, 0]; + function cronexec(output, date) { var m = date.getMinutes(); @@ -28,7 +31,10 @@ function cronexec(output, date) { break; } } else if (m.type === 'every') { - if (m.value[i] !== '*' && m.value[i] % val !== 0) { + // m.value = [start, step] where the start is "*" or a number: "*/15" or "5/15" + var step = m.value[1]; + var start = m.value[0] === '*' ? CRON_MIN[i] : m.value[0]; + if (!step || val < start || (val - start) % step !== 0) { is = false; break; } diff --git a/global.js b/global.js index 3a24c17..203180c 100755 --- a/global.js +++ b/global.js @@ -199,7 +199,7 @@ global.AJAX = function(url, data, callback) { switch (type.toLowerCase()) { case 'text/xml': case 'application/xml': - value = response.body ? response.body.parseXML(self.$replace ? true : false) : {}; + value = response.body ? DEF.parsers.xml(response.body) : {}; break; case 'application/x-www-form-urlencoded': value = response.body ? DEF.parsers.urlencoded(response.body) : {}; @@ -305,7 +305,7 @@ global.setTimeout2 = function(id, callback, timeout, limit, a, b, c, d, e) { clearTimeout(cache.timer); cache.count++; } else - cache = internal[key] = {}; + cache = internal[key] = { count: 0 }; cache.callback = callback; cache.timer = setTimeout(timeout2, timeout, key, a, b, c, d, e); diff --git a/index.js b/index.js index 0e72215..4ab645f 100755 --- a/index.js +++ b/index.js @@ -1017,10 +1017,22 @@ F.import = function(url, callback) { F.download(url, filename, function(err, response) { var m; + + // An error page is not a module + if (!err && response && response.status >= 400) { + err = new Error('The address answered with the status ' + response.status); + F.Fs.unlink(filename, NOOP); + } + if (!err) { - m = require(filename); - F.temporary.dependencies[url] = 1; + try { + m = require(filename); + F.temporary.dependencies[url] = 1; + } catch (e) { + err = e; + } } + callback && callback(err, m, response); }); }; @@ -1087,14 +1099,14 @@ F.cleanup = function(stream, callback) { F.python = function(filename, callback) { if (!callback) - return new Promise((resolve, reject) => F.python(filename, (err, response) => err ? reject(err) : response)); + return new Promise((resolve, reject) => F.python(filename, (err, response) => err ? reject(err) : resolve(response))); F.Child.exec(F.config.$python + ' ' + filename, { cwd: F.Path.dirname(filename) }, callback); }; F.pipinstall = function(name, callback) { if (!callback) - return new Promise((resolve, reject) => F.npminstall(name, err => err ? reject(err) : resolve())); + return new Promise((resolve, reject) => F.pipinstall(name, err => err ? reject(err) : resolve())); var args = {}; args.cwd = F.directory; @@ -1255,31 +1267,36 @@ F.loadservices = function() { return; } - let index = 0; + // One pass: the requests that are still alive are moved to the front of the same array (splice() moves the rest of the array each time) + let pending = F.temporary.pending; + let count = 0; - while (true) { - let ctrl = F.temporary.pending[index]; - if (ctrl) { - if (ctrl.destroyed || ctrl.res.headersSent || ctrl.res.writableEnded) { - F.temporary.pending.splice(index, 1); - } else if (ctrl.timeout <= 0) { + for (let index = 0; index < pending.length; index++) { - F.stats.response.timeout++; + let ctrl = pending[index]; - if (F.timeouts.push((NOW = new Date()).toJSON() + ' ' + ctrl.url) > 5) - F.timeouts.shift(); + if (!ctrl) + continue; - ctrl.fallback(408); - F.temporary.pending.splice(index, 1); - } else { - F.stats.request.pending++; - ctrl.timeout -= 5; // 5 seconds - index++; - } - } else - break; + if (ctrl.destroyed || ctrl.res.headersSent || ctrl.res.writableEnded) + continue; + + if (ctrl.timeout <= 0) { + + F.stats.response.timeout++; + + if (F.timeouts.push((NOW = new Date()).toJSON() + ' ' + ctrl.url) > 5) + F.timeouts.shift(); + + ctrl.fallback(408); + continue; + } + + ctrl.timeout -= 5; // 5 seconds + pending[count++] = ctrl; } + pending.length = count; F.stats.request.pending = F.temporary.pending.length; }, 5000); @@ -2737,7 +2754,7 @@ F.logmail = function(email, subject, body, callback) { if (!subject) subject = F.config.name; - var body = '' + subject + '
' + (typeof(body) === 'object' ? JSON.stringify(body).escape() : body) + '
'; + var body = '' + subject + '
' + (typeof(body) === 'object' ? JSON.stringify(body).encode() : body) + '
'; return F.def.onMail(email, subject, body, callback); }; diff --git a/ldap.js b/ldap.js index 54a0a4f..3b637cd 100644 --- a/ldap.js +++ b/ldap.js @@ -240,8 +240,9 @@ Reader.prototype.parse = function() { obj.dn = self.readstring(); obj.error = self.readstring(); - if (obj.error) { - self.callback(obj.error, null); + // A bind that was not successful (the status is not 0) ends here, also when the server sends no diagnostic text (OpenLDAP: 49) + if (obj.error || obj.status) { + self.callback(obj.error || ('LDAP bind error ' + obj.status), null); return; } diff --git a/proxy.js b/proxy.js index fb2df0e..c2c4639 100644 --- a/proxy.js +++ b/proxy.js @@ -15,10 +15,10 @@ BUFFER_END.writeUInt8(2); const BUFFER_ERROR = Buffer.alloc(1); - BUFFER_END.writeUInt8(3); + BUFFER_ERROR.writeUInt8(3); const BUFFER_ABORT = Buffer.alloc(1); - BUFFER_END.writeUInt8(4); + BUFFER_ABORT.writeUInt8(4); const REG_ANSI = /\x1B\[[0-?]*[ -/]*[@-~]/g; const Pending = {}; diff --git a/querybuilder.js b/querybuilder.js index 7a91f8c..44489e7 100644 --- a/querybuilder.js +++ b/querybuilder.js @@ -920,7 +920,7 @@ QBP.gridfields = function(fields, allowed) { } } - if (!count) + if (count) t.options.fields = newfields; return t; diff --git a/tests/common/fixes.js b/tests/common/fixes.js new file mode 100644 index 0000000..7d10c88 --- /dev/null +++ b/tests/common/fixes.js @@ -0,0 +1,158 @@ +/* eslint-disable */ +require('../../index'); // Total.js framework v5 +require('../../test'); // Unit test module + +CONF.$imprint = false; +CONF.$python = 'node'; + + +ROUTE('GET /xml/', $ => $.content('Total.js', 'application/xml')); +ROUTE('GET /module/', $ => $.plain('module.exports = 1;')); +ROUTE('GET /search/', $ => $.json({ href: $.href('page', 2), all: $.href() })); + +var runs = 0; + +NEWACTION('Cached', { + name: 'Cached', + query: 'page:Number', + cache: { expire: '1 minute', query: 'page' }, + action: function($) { + runs++; + $.callback({ runs: runs, page: $.query.page }); + } +}); + +ROUTE('GET /cached/ --> Cached'); + +F.run({ port: 8211, release: true, watcher: false }); + +ON('ready', function() { + + Test.push('Fixes', function(next) { + + var arr = []; + + arr.push(resume => setTimeout(resume, 500)); + + // CRON: the step form + arr.push(function(resume) { + var cron = require('../../cron'); + function hits(line, from, to, fn) { + var check = cron.make(line); + var out = []; + for (var i = from; i <= to; i++) { + if (check(fn(i))) + out.push(i); + } + return out.join(','); + } + Test.print('CRON */15 matches 0,15,30,45', hits('*/15 * * * *', 0, 59, i => new Date(2026, 0, 5, 10, i)) === '0,15,30,45' ? null : 'The step form does not match the expected minutes'); + Test.print('CRON 5/20 starts at 5', hits('5/20 * * * *', 0, 59, i => new Date(2026, 0, 5, 10, i)) === '5,25,45' ? null : 'The step form with a start does not match the expected minutes'); + Test.print('CRON */6 in the hours', hits('0 */6 * * *', 0, 23, i => new Date(2026, 0, 5, i, 0)) === '0,6,12,18' ? null : 'The step form does not match the expected hours'); + Test.print('CRON */10 in the days', hits('0 0 */10 * *', 1, 31, i => new Date(2026, 0, i, 0, 0)) === '1,11,21,31' ? null : 'The step form does not match the expected days'); + resume(); + }); + + // setTimeout2: the limit + arr.push(function(resume) { + var count = 0; + var fn = () => count++; + for (var i = 0; i < 4; i++) + setTimeout2('fixes-limit', fn, 5000, 2); + Test.print('setTimeout2 limit runs the callback at once', count === 1 ? null : 'Expected the callback after the limit, got ' + count); + clearTimeout2('fixes-limit'); + resume(); + }); + + // AJAX: an answer of the type XML + arr.push(function(resume) { + var timer = setTimeout(() => Test.print('AJAX with XML answers', 'The callback was not called'), 3000); + AJAX('GET http://127.0.0.1:8211/xml/', function(err, response) { + clearTimeout(timer); + Test.print('AJAX with XML answers', !err && response && response.root_name === 'Total.js' ? null : 'Expected the parsed XML, got ' + JSON.stringify(err || response)); + resume(); + }); + }); + + // IMPORT: an address that answers 404 + arr.push(function(resume) { + var timer = setTimeout(() => Test.print('IMPORT with 404', 'The callback was not called'), 3000); + IMPORT('http://127.0.0.1:8211/missing/', function(err) { + clearTimeout(timer); + Test.print('IMPORT with 404 gives an error', err ? null : 'Expected an error'); + resume(); + }); + }); + + arr.push(function(resume) { + IMPORT('http://127.0.0.1:8211/module/', function(err, module) { + Test.print('IMPORT of a module', !err && module === 1 ? null : 'Expected the module'); + resume(); + }); + }); + + // F.python without a callback + arr.push(function(resume) { + var file = PATH.tmp('fixes-python.js'); + require('fs').writeFileSync(file, 'console.log("python-ok");'); + var timer = setTimeout(() => Test.print('F.python as a promise', 'The promise was not settled'), 3000); + F.python(file).then(function(response) { + clearTimeout(timer); + Test.print('F.python as a promise', response && response.indexOf('python-ok') !== -1 ? null : 'Expected the output'); + resume(); + }); + }); + + // LOGMAIL: an object as the body + arr.push(function(resume) { + var old = DEF.onMail; + var body = ''; + DEF.onMail = function(email, subject, html) { + body = html; + }; + LOGMAIL('nobody@example.com', { rows: '<1>' }); + DEF.onMail = old; + Test.print('LOGMAIL with an object', body.indexOf('{"rows"') !== -1 || body.indexOf('<1>') !== -1 ? null : 'Expected the object in the body'); + resume(); + }); + + // $.href + arr.push(function(resume) { + RESTBuilder.GET('http://127.0.0.1:8211/search/?q=a&page=1').exec(function(err, response) { + Test.print('$.href()', !err && response && response.href && response.href.indexOf('page=2') !== -1 ? null : 'Expected a link with page=2, got ' + JSON.stringify(err || response)); + resume(); + }); + }); + + // The cache option of an action + arr.push(function(resume) { + RESTBuilder.GET('http://127.0.0.1:8211/cached/?page=1').exec(function(err, first) { + RESTBuilder.GET('http://127.0.0.1:8211/cached/?page=1').exec(function(err2, second) { + RESTBuilder.GET('http://127.0.0.1:8211/cached/?page=2').exec(function(err3, third) { + Test.print('Action cache: the same key is answered from the cache', first && second && first.runs === 1 && second.runs === 1 ? null : 'Expected one run for the same page'); + Test.print('Action cache: another key runs the action', third && third.runs === 2 ? null : 'Expected a new run for another page'); + resume(); + }); + }); + }); + }); + + // Compact types JSON and IP + arr.push(function(resume) { + var js = 'json:JSON,address:IP'.toJSONSchema(); + var bad = js.transform({ json: '{broken', address: '999.1.1.1' }, false, new ErrorBuilder()); + var good = js.transform({ json: '{"a":1}', address: '127.0.0.1' }, false, new ErrorBuilder()); + Test.print('Compact JSON and IP reject invalid values', bad.error ? null : 'Expected an error'); + Test.print('Compact JSON and IP accept valid values', !good.error ? null : 'Expected no error'); + resume(); + }); + + arr.async(function() { + // The files that the test created in the temporary folder + require('fs').rmSync(PATH.tmp(), { recursive: true, force: true }); + next(); + }); + }); + + Test.run(); +}); diff --git a/tests/run.js b/tests/run.js index 57813f9..912423c 100644 --- a/tests/run.js +++ b/tests/run.js @@ -14,7 +14,7 @@ tests.push('routing/index.js'); tests.push('schemas/index.js'); tests.push('server/index.js'); tests.push('staticfiles/index.js'); -// tests.push('common/cron.js'); +tests.push('common/fixes.js'); console.log('=========================================='); console.log('Total.js v5 unit tests'); diff --git a/utils.js b/utils.js index f6d4201..49a94b8 100755 --- a/utils.js +++ b/utils.js @@ -73,7 +73,7 @@ const REG_TIME = /am|pm/i; const REG_XMLKEY = /\[|\]|:|\.|_/g; const REG_HEADERPARSER = /(name|filename)=".*?"|content-type:\s[a-z0-9-./+]+/ig; const HEADEREND = Buffer.from('\r\n\r\n', 'ascii'); -const JSCHEMAS_NULLABLE = { json: 1, base64: 1, guid: 1, datauri: 1, datauri2: 2, uid: 1, string2: 1 }; +const JSCHEMAS_NULLABLE = { json: 1, ip: 1, base64: 1, guid: 1, datauri: 1, datauri2: 2, uid: 1, string2: 1 }; const ARR_UA_OS = ['android', 'windows', 'ios', 'iphone', 'ubuntu', 'fedora', 'linux', 'mac os']; const ARR_UA_BROWSER = ['xbox', 'playstation', 'nintendo', 'appletv', 'fxios', 'opera', 'edge', 'vivaldi', 'chrome', 'firefox', 'kindle', 'safari']; @@ -6244,6 +6244,8 @@ SP.toJSONSchema = SP.parseSchema = function(name, url) { case 'capitalize2': case 'color': case 'icon': + case 'json': + case 'ip': case 'base64': case 'datauri': case 'datauri2': From 889cda0b990458defe2ecce20369b82bff3180c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gera=20G=2E=20G=C3=BCiles?= Date: Mon, 19 Oct 2026 13:29:39 +0200 Subject: [PATCH 2/5] Pending requests: at most 5 lines of the log of timeouts for each pass, with one date, instead of a date and a string for every request that timed out --- index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 4ab645f..62f3cb1 100755 --- a/index.js +++ b/index.js @@ -1270,6 +1270,7 @@ F.loadservices = function() { // One pass: the requests that are still alive are moved to the front of the same array (splice() moves the rest of the array each time) let pending = F.temporary.pending; let count = 0; + let logged = 0; for (let index = 0; index < pending.length; index++) { @@ -1285,8 +1286,14 @@ F.loadservices = function() { F.stats.response.timeout++; - if (F.timeouts.push((NOW = new Date()).toJSON() + ' ' + ctrl.url) > 5) - F.timeouts.shift(); + // Only the last 5 timeouts are kept, so a pass writes at most 5 lines (many timeouts do not create a string for each request) + if (logged < 5) { + if (!logged) + NOW = new Date(); + logged++; + if (F.timeouts.push(NOW.toJSON() + ' ' + ctrl.url) > 5) + F.timeouts.shift(); + } ctrl.fallback(408); continue; From b4a667828df0882fb6fad99f5600960f3f3c30d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gera=20G=2E=20G=C3=BCiles?= Date: Mon, 19 Oct 2026 15:48:08 +0200 Subject: [PATCH 3/5] Timeouts log: the date and the address are kept as they are and the date is serialized only when the stats are read --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 62f3cb1..2291cfa 100755 --- a/index.js +++ b/index.js @@ -1286,12 +1286,12 @@ F.loadservices = function() { F.stats.response.timeout++; - // Only the last 5 timeouts are kept, so a pass writes at most 5 lines (many timeouts do not create a string for each request) + // Only the last 5 timeouts are kept, so a pass stores at most 5 of them (the date is serialized only when the stats are read) if (logged < 5) { if (!logged) NOW = new Date(); logged++; - if (F.timeouts.push(NOW.toJSON() + ' ' + ctrl.url) > 5) + if (F.timeouts.push({ date: NOW, url: ctrl.url }) > 5) F.timeouts.shift(); } @@ -2902,7 +2902,7 @@ F.loadstats = function() { let timeout = F.timeouts[F.timeouts.length - 1]; stats.lasterror = err ? (err.date.toJSON() + ' ' + (err.name ? (err.name + ' - ') : '') + err.error) : undefined; - stats.lasttimeout = timeout; + stats.lasttimeout = timeout ? (timeout.date.toJSON() + ' ' + timeout.url) : undefined; if ((stats.usage > 80 || stats.memory > 600 || stats.pending > 1000) && lastwarning !== NOW.getHours()) { lastwarning = NOW.getHours(); From 22be0239ad36cb0f2981cf02f1df8df70b778450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gera=20G=2E=20G=C3=BCiles?= Date: Mon, 19 Oct 2026 15:56:11 +0200 Subject: [PATCH 4/5] Mail attachments: attachmentfs() reads the file of the storage as a Buffer, and attachment() with a Buffer encodes the whole Buffer to Base64 before splitting it into lines; test with a tiny SMTP server --- changelog.txt | 2 + mail.js | 26 ++---- tests/common/mail-attachments.js | 149 +++++++++++++++++++++++++++++++ tests/run.js | 1 + 4 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 tests/common/mail-attachments.js diff --git a/changelog.txt b/changelog.txt index 14a42fa..dcfdae0 100644 --- a/changelog.txt +++ b/changelog.txt @@ -25,6 +25,8 @@ - fixed `LDAP()`: a bind with a status other than 0 gives an error even when the server sends no text - added support for the `cache` option of `NEWACTION()` (the response is kept for the same `query`, `params`, `input`, `user` or `language`) - added the compact schema types `json` and `ip` (the values are validated) +- fixed `attachmentfs()` in the mail: the file of the storage is read as a Buffer and the message is completed (it threw a `TypeError` and the SMTP connection stayed open) +- fixed `attachment()` with a `Buffer` in the mail: the Base64 of the whole Buffer is split into lines (the bytes were split into pieces of 68 bytes and each one got its own padding) ======================== 0.0.19 diff --git a/mail.js b/mail.js index d3d691f..a17f838 100755 --- a/mail.js +++ b/mail.js @@ -150,7 +150,7 @@ Message.prototype.attachmentfs = function(storage, id, name, contentid) { if (name) { ext = F.TUtils.getExtension(name); - type = F.TUtils.contentTypes[ext]; + type = F.TUtils.contentTypes[ext] || 'application/octet-stream'; } var obj = {}; @@ -328,7 +328,7 @@ Mailer.$writeattachment = function(obj) { var stream; if (attachment.storage) { - FILESTORAGE(attachment.storage).readbase64(attachment.filename, function(err, meta) { + FILESTORAGE(attachment.storage).readbuffer(attachment.filename, function(err, buffer, meta) { if (err) { F.error(err, 'Mail.filestorage()', attachment.filename); Mailer.$writeattachment(obj); @@ -336,11 +336,11 @@ Mailer.$writeattachment = function(obj) { if (!attachment.name) { attachment.name = meta.name; - attachment.type = meta.type; + attachment.type = meta.type || 'application/octet-stream'; attachment.extension = meta.ext; } - writeattachemnt_stream(attachment, obj, meta.stream); + writeattachemnt_stream(attachment, obj, buffer); } }); } else { @@ -413,20 +413,12 @@ function writeattachmentbytes(chunk) { function writeattachmentbytesinline(mailerdata, chunk) { - var length = chunk.length; - var count = 0; - var beg = 0; - - while (count < length) { - - count += 68; - - if (count > length) - count = length; + // The Base64 of the whole Buffer is split into lines (splitting the bytes would add padding to each line) + var value = chunk.toString('base64'); + var length = value.length; - Mailer.$writeline(mailerdata, chunk.slice(beg, count).toString('base64')); - beg = count; - } + for (var index = 0; index < length; index += 68) + Mailer.$writeline(mailerdata, value.substring(index, index + 68)); } Mailer.try = function(options, callback) { diff --git a/tests/common/mail-attachments.js b/tests/common/mail-attachments.js new file mode 100644 index 0000000..0562356 --- /dev/null +++ b/tests/common/mail-attachments.js @@ -0,0 +1,149 @@ +/* eslint-disable */ +require('../../index'); // Total.js framework v5 +require('../../test'); // Unit test module + +CONF.$imprint = false; +CONF.mail_from = 'from@example.com'; +CONF.mail_from_name = 'Test'; + +F.run({ port: 8221, release: true, watcher: false }); + +// The SMTP settings of the application: the tiny server below +ON('ready', () => F.config.smtp = { server: '127.0.0.1', port: 2529 }); + +// A tiny SMTP server that records what it receives, standing in for a real one +const net = require('net'); +const received = []; + +const server = net.createServer(function(socket) { + let data = false; + let buffer = ''; + let current = ''; + socket.write('220 localhost ready\r\n'); + socket.on('data', function(chunk) { + buffer += chunk.toString('utf8'); + let index; + while ((index = buffer.indexOf('\r\n')) !== -1) { + const line = buffer.substring(0, index); + buffer = buffer.substring(index + 2); + if (data) { + if (line === '.') { + data = false; + received.push(current); + current = ''; + socket.write('250 queued\r\n'); + } else + current += line + '\n'; + continue; + } + const command = line.substring(0, 4).toUpperCase(); + if (command === 'EHLO' || command === 'HELO') + socket.write('250 localhost\r\n'); + else if (command === 'DATA') { + data = true; + socket.write('354 go on\r\n'); + } else if (command === 'QUIT') { + socket.write('221 bye\r\n'); + socket.end(); + } else + socket.write('250 ok\r\n'); + } + }); +}); + +// The attachment with the name: its Base64 is decoded and the lines of the Base64 must be valid +function attachment(raw, name) { + const parts = raw.split(/\n--/); + for (const part of parts) { + if (part.indexOf('filename="' + name + '"') === -1) + continue; + const body = part.split('Content-Transfer-Encoding: base64\n\n')[1]; + const lines = body.split('\n').map(m => m.trim()).filter(m => m && m.indexOf('--') !== 0); + return { lines: lines, data: Buffer.from(lines.join(''), 'base64'), valid: lines.every((m, i) => /^[A-Za-z0-9+/]*={0,2}$/.test(m) && (i === lines.length - 1 || m.indexOf('=') === -1)) }; + } + return null; +} + +function send(prepare) { + return new Promise(function(resolve) { + const before = received.length; + const message = DEF.onMail('to@example.com', 'Attachment test', '

Hello

', function(err) { + setTimeout(function() { + resolve({ err: err, raw: received[before] || null }); + }, 300); + }); + prepare(message); + }); +} + +ON('ready', function() { + + // All the byte values, a size that is not a multiple of 3 and of 68 + const content = Buffer.alloc(100003); + for (let i = 0; i < content.length; i++) + content[i] = (i * 31 + (i >> 8)) & 255; + + Test.push('Mail attachments', function(next) { + + const arr = []; + + arr.push(function(resume) { + server.listen(2529, '127.0.0.1', resume); + }); + + arr.push(function(resume) { + send(m => m.attachment(content, 'buffer.bin')).then(function(r) { + const a = r.raw && attachment(r.raw, 'buffer.bin'); + Test.print('attachment() with a Buffer is valid Base64', a && a.valid ? null : 'The Base64 has padding inside'); + Test.print('attachment() with a Buffer is sent without changes', a && a.data.equals(content) ? null : 'The attachment is corrupted'); + resume(); + }); + }); + + arr.push(function(resume) { + const file = PATH.tmp('mail-attachment.bin'); + require('fs').writeFileSync(file, content); + send(m => m.attachment(file, 'file.bin')).then(function(r) { + const a = r.raw && attachment(r.raw, 'file.bin'); + Test.print('attachment() with a file is sent without changes', a && a.valid && a.data.equals(content) ? null : 'The attachment is corrupted'); + resume(); + }); + }); + + arr.push(function(resume) { + FILESTORAGE('mailtest').save('f1', 'stored.bin', content, function(err) { + send(m => m.attachmentfs('mailtest', 'f1', 'stored.bin')).then(function(r) { + const a = r.raw && attachment(r.raw, 'stored.bin'); + Test.print('attachmentfs() sends the file of the storage', a && a.valid && a.data.equals(content) ? null : 'The email was not sent with the attachment'); + resume(); + }); + }); + }); + + arr.push(function(resume) { + FILESTORAGE('mailtest').save('f2', 'noname.bin', content, function(err) { + send(m => m.attachmentfs('mailtest', 'f2')).then(function(r) { + const a = r.raw && attachment(r.raw, 'noname.bin'); + Test.print('attachmentfs() takes the name of the file when there is no name', a && a.data.equals(content) ? null : 'The name was not taken from the storage'); + resume(); + }); + }); + }); + + arr.push(function(resume) { + send(m => m.attachmentfs('mailtest', 'missing')).then(function(r) { + Test.print('attachmentfs() with a missing file finishes the message', r.raw && r.raw.indexOf('Attachment test') !== -1 || r.raw ? null : 'The message was not completed'); + resume(); + }); + }); + + arr.async(function() { + server.close(); + require('fs').rmSync(PATH.tmp(), { recursive: true, force: true }); + require('fs').rmSync(PATH.databases('fs-mailtest'), { recursive: true, force: true }); + next(); + }); + }); + + Test.run(); +}); diff --git a/tests/run.js b/tests/run.js index 912423c..7606e32 100644 --- a/tests/run.js +++ b/tests/run.js @@ -7,6 +7,7 @@ tests.push('aimodel.js'); // tests.push('common/utils.js'); // tests.push('tms/index.js'); tests.push('common/htmlparser.js'); +tests.push('common/mail-attachments.js'); tests.push('minificators/index.js'); tests.push('nosql/index.js'); tests.push('proxy/index.js'); From b2d2ab05859e491da543d0218730e353e11b868a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gera=20G=2E=20G=C3=BCiles?= Date: Mon, 19 Oct 2026 16:03:29 +0200 Subject: [PATCH 5/5] Timeouts log: back to the original code (F.timeouts keeps strings), only the loop of the pending requests changes --- index.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 2291cfa..4ab645f 100755 --- a/index.js +++ b/index.js @@ -1270,7 +1270,6 @@ F.loadservices = function() { // One pass: the requests that are still alive are moved to the front of the same array (splice() moves the rest of the array each time) let pending = F.temporary.pending; let count = 0; - let logged = 0; for (let index = 0; index < pending.length; index++) { @@ -1286,14 +1285,8 @@ F.loadservices = function() { F.stats.response.timeout++; - // Only the last 5 timeouts are kept, so a pass stores at most 5 of them (the date is serialized only when the stats are read) - if (logged < 5) { - if (!logged) - NOW = new Date(); - logged++; - if (F.timeouts.push({ date: NOW, url: ctrl.url }) > 5) - F.timeouts.shift(); - } + if (F.timeouts.push((NOW = new Date()).toJSON() + ' ' + ctrl.url) > 5) + F.timeouts.shift(); ctrl.fallback(408); continue; @@ -2902,7 +2895,7 @@ F.loadstats = function() { let timeout = F.timeouts[F.timeouts.length - 1]; stats.lasterror = err ? (err.date.toJSON() + ' ' + (err.name ? (err.name + ' - ') : '') + err.error) : undefined; - stats.lasttimeout = timeout ? (timeout.date.toJSON() + ' ' + timeout.url) : undefined; + stats.lasttimeout = timeout; if ((stats.usage > 80 || stats.memory > 600 || stats.pending > 1000) && lastwarning !== NOW.getHours()) { lastwarning = NOW.getHours();