deno.land / std@0.224.0 / toml / _parser.ts
_parser.ts
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.// This module is browser compatible.
import { deepMerge } from "../collections/deep_merge.ts";
// ---------------------------// Interfaces and base classes// ---------------------------
interface Success<T> { ok: true; body: T;}interface Failure { ok: false;}type ParseResult<T> = Success<T> | Failure;
type ParserComponent<T = unknown> = (scanner: Scanner) => ParseResult<T>;
type BlockParseResultBody = { type: "Block"; value: Record<string, unknown>;} | { type: "Table"; key: string[]; value: Record<string, unknown>;} | { type: "TableArray"; key: string[]; value: Record<string, unknown>;};
export class TOMLParseError extends Error {}
export class Scanner { #whitespace = /[ \t]/; #position = 0; constructor(private source: string) {}
/** * Get current character * @param index - relative index from current position */ char(index = 0) { return this.source[this.#position + index] ?? ""; }
/** * Get sliced string * @param start - start position relative from current position * @param end - end position relative from current position */ slice(start: number, end: number): string { return this.source.slice(this.#position + start, this.#position + end); }
/** * Move position to next */ next(count?: number) { if (typeof count === "number") { for (let i = 0; i < count; i++) { this.#position++; } } else { this.#position++; } }
/** * Move position until current char is not a whitespace, EOL, or comment. * @param options.inline - skip only whitespaces */ nextUntilChar( options: { inline?: boolean; comment?: boolean } = { comment: true }, ) { if (options.inline) { while (this.#whitespace.test(this.char()) && !this.eof()) { this.next(); } } else { while (!this.eof()) { const char = this.char(); if (this.#whitespace.test(char) || this.isCurrentCharEOL()) { this.next(); } else if (options.comment && this.char() === "#") { // entering comment while (!this.isCurrentCharEOL() && !this.eof()) { this.next(); } } else { break; } } } // Invalid if current char is other kinds of whitespace if (!this.isCurrentCharEOL() && /\s/.test(this.char())) { const escaped = "\\u" + this.char().charCodeAt(0).toString(16); throw new TOMLParseError(`Contains invalid whitespaces: \`${escaped}\``); } }
/** * Position reached EOF or not */ eof() { return this.position() >= this.source.length; }
/** * Get current position */ position() { return this.#position; }
isCurrentCharEOL() { return this.char() === "\n" || this.slice(0, 2) === "\r\n"; }}
// -----------------------// Utilities// -----------------------
function success<T>(body: T): Success<T> { return { ok: true, body, };}function failure(): Failure { return { ok: false, };}
export const Utils = { unflat( keys: string[], values: unknown = {}, cObj?: unknown, ): Record<string, unknown> { const out: Record<string, unknown> = {}; if (keys.length === 0) { return cObj as Record<string, unknown>; } else { if (!cObj) { cObj = values; } const key: string | undefined = keys[keys.length - 1]; if (typeof key === "string") { out[key] = cObj; } return this.unflat(keys.slice(0, -1), values, out); } }, deepAssignWithTable(target: Record<string, unknown>, table: { type: "Table" | "TableArray"; key: string[]; value: Record<string, unknown>; }) { if (table.key.length === 0 || table.key[0] == null) { throw new Error("Unexpected key length"); } const value = target[table.key[0]];
if (typeof value === "undefined") { Object.assign( target, this.unflat( table.key, table.type === "Table" ? table.value : [table.value], ), ); } else if (Array.isArray(value)) { if (table.type === "TableArray" && table.key.length === 1) { value.push(table.value); } else { const last = value[value.length - 1]; Utils.deepAssignWithTable(last, { type: table.type, key: table.key.slice(1), value: table.value, }); } } else if (typeof value === "object" && value !== null) { Utils.deepAssignWithTable(value as Record<string, unknown>, { type: table.type, key: table.key.slice(1), value: table.value, }); } else { throw new Error("Unexpected assign"); } },};
// ---------------------------------// Parser combinators and generators// ---------------------------------
function or<T>(parsers: ParserComponent<T>[]): ParserComponent<T> { return function Or(scanner: Scanner): ParseResult<T> { for (const parse of parsers) { const result = parse(scanner); if (result.ok) { return result; } } return failure(); };}
function join<T>( parser: ParserComponent<T>, separator: string,): ParserComponent<T[]> { const Separator = character(separator); return function Join(scanner: Scanner): ParseResult<T[]> { const first = parser(scanner); if (!first.ok) { return failure(); } const out: T[] = [first.body]; while (!scanner.eof()) { if (!Separator(scanner).ok) { break; } const result = parser(scanner); if (result.ok) { out.push(result.body); } else { throw new TOMLParseError(`Invalid token after "${separator}"`); } } return success(out); };}
function kv<T>( keyParser: ParserComponent<string[]>, separator: string, valueParser: ParserComponent<T>,): ParserComponent<{ [key: string]: unknown }> { const Separator = character(separator); return function Kv( scanner: Scanner, ): ParseResult<{ [key: string]: unknown }> { const key = keyParser(scanner); if (!key.ok) { return failure(); } const sep = Separator(scanner); if (!sep.ok) { throw new TOMLParseError(`key/value pair doesn't have "${separator}"`); } const value = valueParser(scanner); if (!value.ok) { throw new TOMLParseError( `Value of key/value pair is invalid data format`, ); } return success(Utils.unflat(key.body, value.body)); };}
function merge( parser: ParserComponent<unknown[]>,): ParserComponent<Record<string, unknown>> { return function Merge( scanner: Scanner, ): ParseResult<Record<string, unknown>> { const result = parser(scanner); if (!result.ok) { return failure(); } let body = {}; for (const record of result.body) { if (typeof body === "object" && body !== null) { // deno-lint-ignore no-explicit-any body = deepMerge(body, record as Record<any, any>); } } return success(body); };}
function repeat<T>( parser: ParserComponent<T>,): ParserComponent<T[]> { return function Repeat( scanner: Scanner, ) { const body: T[] = []; while (!scanner.eof()) { const result = parser(scanner); if (result.ok) { body.push(result.body); } else { break; } scanner.nextUntilChar(); } if (body.length === 0) { return failure(); } return success(body); };}
function surround<T>( left: string, parser: ParserComponent<T>, right: string,): ParserComponent<T> { const Left = character(left); const Right = character(right); return function Surround(scanner: Scanner) { if (!Left(scanner).ok) { return failure(); } const result = parser(scanner); if (!result.ok) { throw new TOMLParseError(`Invalid token after "${left}"`); } if (!Right(scanner).ok) { throw new TOMLParseError( `Not closed by "${right}" after started with "${left}"`, ); } return success(result.body); };}
function character(str: string) { return function character(scanner: Scanner): ParseResult<void> { scanner.nextUntilChar({ inline: true }); if (scanner.slice(0, str.length) === str) { scanner.next(str.length); } else { return failure(); } scanner.nextUntilChar({ inline: true }); return success(undefined); };}
// -----------------------// Parser components// -----------------------
const Patterns = { BARE_KEY: /[A-Za-z0-9_-]/, FLOAT: /[0-9_\.e+\-]/i, END_OF_VALUE: /[ \t\r\n#,}\]]/,};
export function BareKey(scanner: Scanner): ParseResult<string> { scanner.nextUntilChar({ inline: true }); if (!scanner.char() || !Patterns.BARE_KEY.test(scanner.char())) { return failure(); } const acc: string[] = []; while (scanner.char() && Patterns.BARE_KEY.test(scanner.char())) { acc.push(scanner.char()); scanner.next(); } const key = acc.join(""); return success(key);}
function EscapeSequence(scanner: Scanner): ParseResult<string> { if (scanner.char() === "\\") { scanner.next(); // See https://toml.io/en/v1.0.0-rc.3#string switch (scanner.char()) { case "b": scanner.next(); return success("\b"); case "t": scanner.next(); return success("\t"); case "n": scanner.next(); return success("\n"); case "f": scanner.next(); return success("\f"); case "r": scanner.next(); return success("\r"); case "u": case "U": { // Unicode character const codePointLen = scanner.char() === "u" ? 4 : 6; const codePoint = parseInt( "0x" + scanner.slice(1, 1 + codePointLen), 16, ); const str = String.fromCodePoint(codePoint); scanner.next(codePointLen + 1); return success(str); } case '"': scanner.next(); return success('"'); case "\\": scanner.next(); return success("\\"); default: throw new TOMLParseError( `Invalid escape sequence: \\${scanner.char()}`, ); } } else { return failure(); }}
export function BasicString(scanner: Scanner): ParseResult<string> { scanner.nextUntilChar({ inline: true }); if (scanner.char() === '"') { scanner.next(); } else { return failure(); } const acc = []; while (scanner.char() !== '"' && !scanner.eof()) { if (scanner.char() === "\n") { throw new TOMLParseError("Single-line string cannot contain EOL"); } const escapedChar = EscapeSequence(scanner); if (escapedChar.ok) { acc.push(escapedChar.body); } else { acc.push(scanner.char()); scanner.next(); } } if (scanner.eof()) { throw new TOMLParseError( `Single-line string is not closed:\n${acc.join("")}`, ); } scanner.next(); // skip last '"" return success(acc.join(""));}
export function LiteralString(scanner: Scanner): ParseResult<string> { scanner.nextUntilChar({ inline: true }); if (scanner.char() === "'") { scanner.next(); } else { return failure(); } const acc: string[] = []; while (scanner.char() !== "'" && !scanner.eof()) { if (scanner.char() === "\n") { throw new TOMLParseError("Single-line string cannot contain EOL"); } acc.push(scanner.char()); scanner.next(); } if (scanner.eof()) { throw new TOMLParseError( `Single-line string is not closed:\n${acc.join("")}`, ); } scanner.next(); // skip last "'" return success(acc.join(""));}
export function MultilineBasicString( scanner: Scanner,): ParseResult<string> { scanner.nextUntilChar({ inline: true }); if (scanner.slice(0, 3) === '"""') { scanner.next(3); } else { return failure(); } if (scanner.char() === "\n") { // The first newline (LF) is trimmed scanner.next(); } else if (scanner.slice(0, 2) === "\r\n") { // The first newline (CRLF) is trimmed scanner.next(2); } const acc: string[] = []; while (scanner.slice(0, 3) !== '"""' && !scanner.eof()) { // line ending backslash if (scanner.slice(0, 2) === "\\\n") { scanner.next(); scanner.nextUntilChar({ comment: false }); continue; } else if (scanner.slice(0, 3) === "\\\r\n") { scanner.next(); scanner.nextUntilChar({ comment: false }); continue; } const escapedChar = EscapeSequence(scanner); if (escapedChar.ok) { acc.push(escapedChar.body); } else { acc.push(scanner.char()); scanner.next(); } }
if (scanner.eof()) { throw new TOMLParseError( `Multi-line string is not closed:\n${acc.join("")}`, ); } // if ends with 4 `"`, push the fist `"` to string if (scanner.char(3) === '"') { acc.push('"'); scanner.next(); } scanner.next(3); // skip last '"""" return success(acc.join(""));}
export function MultilineLiteralString( scanner: Scanner,): ParseResult<string> { scanner.nextUntilChar({ inline: true }); if (scanner.slice(0, 3) === "'''") { scanner.next(3); } else { return failure(); } if (scanner.char() === "\n") { // The first newline (LF) is trimmed scanner.next(); } else if (scanner.slice(0, 2) === "\r\n") { // The first newline (CRLF) is trimmed scanner.next(2); } const acc: string[] = []; while (scanner.slice(0, 3) !== "'''" && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); } if (scanner.eof()) { throw new TOMLParseError( `Multi-line string is not closed:\n${acc.join("")}`, ); } // if ends with 4 `'`, push the fist `'` to string if (scanner.char(3) === "'") { acc.push("'"); scanner.next(); } scanner.next(3); // skip last "'''" return success(acc.join(""));}
const symbolPairs: [string, unknown][] = [ ["true", true], ["false", false], ["inf", Infinity], ["+inf", Infinity], ["-inf", -Infinity], ["nan", NaN], ["+nan", NaN], ["-nan", NaN],];export function Symbols(scanner: Scanner): ParseResult<unknown> { scanner.nextUntilChar({ inline: true }); const found = symbolPairs.find(([str]) => scanner.slice(0, str.length) === str ); if (!found) { return failure(); } const [str, value] = found; scanner.next(str.length); return success(value);}
export const DottedKey = join( or([BareKey, BasicString, LiteralString]), ".",);
export function Integer(scanner: Scanner): ParseResult<number | string> { scanner.nextUntilChar({ inline: true });
// If binary / octal / hex const first2 = scanner.slice(0, 2); if (first2.length === 2 && /0(?:x|o|b)/i.test(first2)) { scanner.next(2); const acc = [first2]; while (/[0-9a-f_]/i.test(scanner.char()) && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); } if (acc.length === 1) { return failure(); } return success(acc.join("")); }
const acc = []; if (/[+-]/.test(scanner.char())) { acc.push(scanner.char()); scanner.next(); } while (/[0-9_]/.test(scanner.char()) && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); }
if (acc.length === 0 || (acc.length === 1 && /[+-]/.test(acc[0]!))) { return failure(); }
const int = parseInt(acc.filter((char) => char !== "_").join("")); return success(int);}
export function Float(scanner: Scanner): ParseResult<number> { scanner.nextUntilChar({ inline: true });
// lookahead validation is needed for integer value is similar to float let position = 0; while ( scanner.char(position) && !Patterns.END_OF_VALUE.test(scanner.char(position)) ) { if (!Patterns.FLOAT.test(scanner.char(position))) { return failure(); } position++; }
const acc = []; if (/[+-]/.test(scanner.char())) { acc.push(scanner.char()); scanner.next(); } while (Patterns.FLOAT.test(scanner.char()) && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); }
if (acc.length === 0) { return failure(); } const float = parseFloat(acc.filter((char) => char !== "_").join("")); if (isNaN(float)) { return failure(); }
return success(float);}
export function DateTime(scanner: Scanner): ParseResult<Date> { scanner.nextUntilChar({ inline: true });
let dateStr = scanner.slice(0, 10); // example: 1979-05-27 if (/^\d{4}-\d{2}-\d{2}/.test(dateStr)) { scanner.next(10); } else { return failure(); }
const acc = []; // example: 1979-05-27T00:32:00Z while (/[ 0-9TZ.:-]/.test(scanner.char()) && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); } dateStr += acc.join(""); const date = new Date(dateStr.trim()); // invalid date if (isNaN(date.getTime())) { throw new TOMLParseError(`Invalid date string "${dateStr}"`); }
return success(date);}
export function LocalTime(scanner: Scanner): ParseResult<string> { scanner.nextUntilChar({ inline: true });
let timeStr = scanner.slice(0, 8); if (/^(\d{2}):(\d{2}):(\d{2})/.test(timeStr)) { scanner.next(8); } else { return failure(); }
const acc = []; if (scanner.char() === ".") { acc.push(scanner.char()); scanner.next(); } else { return success(timeStr); }
while (/[0-9]/.test(scanner.char()) && !scanner.eof()) { acc.push(scanner.char()); scanner.next(); } timeStr += acc.join(""); return success(timeStr);}
export function ArrayValue(scanner: Scanner): ParseResult<unknown[]> { scanner.nextUntilChar({ inline: true });
if (scanner.char() === "[") { scanner.next(); } else { return failure(); }
const array: unknown[] = []; while (!scanner.eof()) { scanner.nextUntilChar(); const result = Value(scanner); if (result.ok) { array.push(result.body); } else { break; } scanner.nextUntilChar({ inline: true }); // may have a next item, but trailing comma is allowed at array if (scanner.char() === ",") { scanner.next(); } else { break; } } scanner.nextUntilChar();
if (scanner.char() === "]") { scanner.next(); } else { throw new TOMLParseError("Array is not closed"); }
return success(array);}
export function InlineTable( scanner: Scanner,): ParseResult<Record<string, unknown>> { scanner.nextUntilChar(); if (scanner.char(1) === "}") { scanner.next(2); return success({}); } const pairs = surround( "{", join(Pair, ","), "}", )(scanner); if (!pairs.ok) { return failure(); } let table = {}; for (const pair of pairs.body) { table = deepMerge(table, pair); } return success(table);}
export const Value = or([ MultilineBasicString, MultilineLiteralString, BasicString, LiteralString, Symbols, DateTime, LocalTime, Float, Integer, ArrayValue, InlineTable,]);
export const Pair = kv(DottedKey, "=", Value);
export function Block( scanner: Scanner,): ParseResult<BlockParseResultBody> { scanner.nextUntilChar(); const result = merge(repeat(Pair))(scanner); if (result.ok) { return success({ type: "Block", value: result.body, }); } else { return failure(); }}
export const TableHeader = surround("[", DottedKey, "]");
export function Table( scanner: Scanner,): ParseResult<BlockParseResultBody> { scanner.nextUntilChar(); const header = TableHeader(scanner); if (!header.ok) { return failure(); } scanner.nextUntilChar(); const block = Block(scanner); return success({ type: "Table", key: header.body, value: block.ok ? block.body.value : {}, });}
export const TableArrayHeader = surround( "[[", DottedKey, "]]",);
export function TableArray( scanner: Scanner,): ParseResult<BlockParseResultBody> { scanner.nextUntilChar(); const header = TableArrayHeader(scanner); if (!header.ok) { return failure(); } scanner.nextUntilChar(); const block = Block(scanner); return success({ type: "TableArray", key: header.body, value: block.ok ? block.body.value : {}, });}
export function Toml( scanner: Scanner,): ParseResult<Record<string, unknown>> { const blocks = repeat(or([Block, TableArray, Table]))(scanner); if (!blocks.ok) { return failure(); } let body = {}; for (const block of blocks.body) { switch (block.type) { case "Block": { body = deepMerge(body, block.value); break; } case "Table": { Utils.deepAssignWithTable(body, block); break; } case "TableArray": { Utils.deepAssignWithTable(body, block); break; } } } return success(body);}
export function ParserFactory<T>(parser: ParserComponent<T>) { return function parse(tomlString: string): T { const scanner = new Scanner(tomlString);
let parsed: ParseResult<T> | null = null; let err: Error | null = null; try { parsed = parser(scanner); } catch (e) { err = e instanceof Error ? e : new Error("[non-error thrown]"); }
if (err || !parsed || !parsed.ok || !scanner.eof()) { const position = scanner.position(); const subStr = tomlString.slice(0, position); const lines = subStr.split("\n"); const row = lines.length; const column = (() => { let count = subStr.length; for (const line of lines) { if (count > line.length) { count -= line.length + 1; } else { break; } } return count; })(); const message = `Parse error on line ${row}, column ${column}: ${ err ? err.message : `Unexpected character: "${scanner.char()}"` }`; throw new TOMLParseError(message); } return parsed.body; };}
std
Version Info
6 months ago
