deno.land / std@0.224.0 / csv / _io.ts
_io.ts
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300// Originally ported from Go:// https://github.com/golang/go/blob/go1.12.5/src/encoding/csv/// Copyright 2011 The Go Authors. All rights reserved. BSD license.// https://github.com/golang/go/blob/master/LICENSE// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.import { assert } from "../assert/assert.ts";
/** Options for {@linkcode parseRecord}. */export interface ReadOptions { /** Character which separates values. * * @default {","} */ separator?: string; /** Character to start a comment. * * Lines beginning with the comment character without preceding whitespace * are ignored. With leading whitespace the comment character becomes part of * the field, even you provide `trimLeadingSpace: true`. * * @default {"#"} */ comment?: string; /** Flag to trim the leading space of the value. * * This is done even if the field delimiter, `separator`, is white space. * * @default {false} */ trimLeadingSpace?: boolean; /** * Allow unquoted quote in a quoted field or non-double-quoted quotes in * quoted field. * * @default {false} */ lazyQuotes?: boolean; /** * Enabling checking number of expected fields for each row. * * If positive, each record is required to have the given number of fields. * If === 0, it will be set to the number of fields in the first row, so that * future rows must have the same field count. * If negative, no check is made and records may have a variable number of * fields. * * If the wrong number of fields is in a row, a `ParseError` is thrown. */ fieldsPerRecord?: number;}
export const defaultReadOptions: ReadOptions = { separator: ",", trimLeadingSpace: false,};
export interface LineReader { readLine(): Promise<string | null>; isEOF(): boolean;}
export async function parseRecord( line: string, reader: LineReader, opt: ReadOptions, startLine: number, lineIndex: number = startLine,): Promise<Array<string> | null> { // line starting with comment character is ignored if (opt.comment && line[0] === opt.comment) { return []; }
assert(opt.separator !== undefined);
let fullLine = line; let quoteError: ParseError | null = null; const quote = '"'; const quoteLen = quote.length; const separatorLen = opt.separator.length; let recordBuffer = ""; const fieldIndexes = [] as number[]; parseField: for (;;) { if (opt.trimLeadingSpace) { line = line.trimStart(); }
if (line.length === 0 || !line.startsWith(quote)) { // Non-quoted string field const i = line.indexOf(opt.separator); let field = line; if (i >= 0) { field = field.substring(0, i); } // Check to make sure a quote does not appear in field. if (!opt.lazyQuotes) { const j = field.indexOf(quote); if (j >= 0) { const col = runeCount( fullLine.slice(0, fullLine.length - line.slice(j).length), ); quoteError = new ParseError( startLine + 1, lineIndex, col, ERR_BARE_QUOTE, ); break parseField; } } recordBuffer += field; fieldIndexes.push(recordBuffer.length); if (i >= 0) { line = line.substring(i + separatorLen); continue parseField; } break parseField; } else { // Quoted string field line = line.substring(quoteLen); for (;;) { const i = line.indexOf(quote); if (i >= 0) { // Hit next quote. recordBuffer += line.substring(0, i); line = line.substring(i + quoteLen); if (line.startsWith(quote)) { // `""` sequence (append quote). recordBuffer += quote; line = line.substring(quoteLen); } else if (line.startsWith(opt.separator)) { // `","` sequence (end of field). line = line.substring(separatorLen); fieldIndexes.push(recordBuffer.length); continue parseField; } else if (0 === line.length) { // `"\n` sequence (end of line). fieldIndexes.push(recordBuffer.length); break parseField; } else if (opt.lazyQuotes) { // `"` sequence (bare quote). recordBuffer += quote; } else { // `"*` sequence (invalid non-escaped quote). const col = runeCount( fullLine.slice(0, fullLine.length - line.length - quoteLen), ); quoteError = new ParseError( startLine + 1, lineIndex, col, ERR_QUOTE, ); break parseField; } } else if (line.length > 0 || !reader.isEOF()) { // Hit end of line (copy all data so far). recordBuffer += line; const r = await reader.readLine(); lineIndex++; line = r ?? ""; // This is a workaround for making this module behave similarly to the encoding/csv/reader.go. fullLine = line; if (r === null) { // Abrupt end of file (EOF or error). if (!opt.lazyQuotes) { const col = runeCount(fullLine); quoteError = new ParseError( startLine + 1, lineIndex, col, ERR_QUOTE, ); break parseField; } fieldIndexes.push(recordBuffer.length); break parseField; } recordBuffer += "\n"; // preserve line feed (This is because TextProtoReader removes it.) } else { // Abrupt end of file (EOF on error). if (!opt.lazyQuotes) { const col = runeCount(fullLine); quoteError = new ParseError( startLine + 1, lineIndex, col, ERR_QUOTE, ); break parseField; } fieldIndexes.push(recordBuffer.length); break parseField; } } } } if (quoteError) { throw quoteError; } const result = [] as string[]; let preIdx = 0; for (const i of fieldIndexes) { result.push(recordBuffer.slice(preIdx, i)); preIdx = i; } return result;}
function runeCount(s: string): number { // Array.from considers the surrogate pair. return Array.from(s).length;}
/** * A ParseError is returned for parsing errors. * Line numbers are 1-indexed and columns are 0-indexed. */export class ParseError extends SyntaxError { /** Line where the record starts*/ startLine: number; /** Line where the error occurred */ line: number; /** Column (rune index) where the error occurred */ column: number | null;
/** Constructs a new instance. */ constructor( start: number, line: number, column: number | null, message: string, ) { super(); this.startLine = start; this.column = column; this.line = line;
if (message === ERR_FIELD_COUNT) { this.message = `record on line ${line}: ${message}`; } else if (start !== line) { this.message = `record on line ${start}; parse error on line ${line}, column ${column}: ${message}`; } else { this.message = `parse error on line ${line}, column ${column}: ${message}`; } }}
export const ERR_BARE_QUOTE = 'bare " in non-quoted-field';export const ERR_QUOTE = 'extraneous or missing " in quoted-field';export const ERR_INVALID_DELIM = "Invalid Delimiter";export const ERR_FIELD_COUNT = "wrong number of fields";
export function convertRowToObject( row: string[], headers: readonly string[], index: number,) { if (row.length !== headers.length) { throw new Error( `Error number of fields line: ${index}\nNumber of fields found: ${headers.length}\nExpected number of fields: ${row.length}`, ); } const out: Record<string, unknown> = {}; for (const [index, header] of headers.entries()) { out[header] = row[index]; } return out;}
/** Options for {@linkcode parse} and {@linkcode CsvParseStream}. */export type ParseResult<ParseOptions, T> = // If `columns` option is specified, the return type is Record type. T extends ParseOptions & { columns: readonly (infer C extends string)[] } ? RecordWithColumn<C>[] // If `skipFirstRow` option is specified, the return type is Record type. : T extends ParseOptions & { skipFirstRow: true } ? Record<string, string | undefined>[] // If `columns` and `skipFirstRow` option is _not_ specified, the return type is string[][]. : T extends ParseOptions & { columns?: undefined; skipFirstRow?: false | undefined } ? string[][] // else, the return type is Record type or string[][]. : Record<string, string | undefined>[] | string[][];
/** * Record type with column type. * * @example * ``` * type RecordWithColumn<"aaa"|"bbb"> => Record<"aaa"|"bbb", string> * type RecordWithColumn<string> => Record<string, string | undefined> * ``` */export type RecordWithColumn<C extends string> = string extends C ? Record<string, string | undefined> : Record<C, string>;
std
Version Info
6 months ago
