Skip to content

Commit 2fe501e

Browse files
wdavidwwdavidw
committed
feat(csv-parse): info bytes_records (fix #446)
1 parent 50b5812 commit 2fe501e

File tree

12 files changed

+246
-111
lines changed

12 files changed

+246
-111
lines changed

‎packages/csv-parse/lib/api/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const boms = {
2828
const transform = function (original_options = {}) {
2929
const info = {
3030
bytes: 0,
31+
bytes_records: 0,
3132
comment_lines: 0,
3233
empty_lines: 0,
3334
invalid_field_length: 0,
@@ -703,6 +704,7 @@ const transform = function (original_options = {}) {
703704
return;
704705
}
705706
}
707+
this.info.bytes_records += this.info.bytes;
706708
push(record);
707709
},
708710
// Return a tuple with the error and the casted value
@@ -890,6 +892,7 @@ const transform = function (original_options = {}) {
890892
const { columns, raw, encoding } = this.options;
891893
return {
892894
...this.__infoDataSet(),
895+
bytes_records: this.info.bytes,
893896
error: this.state.error,
894897
header: columns === true,
895898
index: this.state.record.length,
@@ -899,8 +902,11 @@ const transform = function (original_options = {}) {
899902
__infoField: function () {
900903
const { columns } = this.options;
901904
const isColumns = Array.isArray(columns);
905+
// Bytes records are only incremented when all records'fields are parsed
906+
const bytes_records = this.info.bytes_records;
902907
return {
903908
...this.__infoRecord(),
909+
bytes_records: bytes_records,
904910
column:
905911
isColumns === true
906912
? columns.length > this.state.record.length

‎packages/csv-parse/lib/index.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ export interface Info {
5959

6060
export interface CastingContext {
6161
readonly column: number | string;
62+
readonly bytes: number;
63+
readonly bytes_records: number;
6264
readonly empty_lines: number;
6365
readonly error: CsvError;
6466
readonly header: boolean;

‎packages/csv-parse/samples/option.cast.context.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ assert.deepStrictEqual(records, [
2727
"2000-01-01T05:00:00.000Z",
2828
{
2929
bytes: 16,
30+
bytes_records: 0,
3031
comment_lines: 0,
3132
empty_lines: 0,
3233
invalid_field_length: 0,
@@ -45,6 +46,7 @@ assert.deepStrictEqual(records, [
4546
"2050-11-27T05:00:00.000Z",
4647
{
4748
bytes: 33,
49+
bytes_records: 17,
4850
comment_lines: 0,
4951
empty_lines: 0,
5052
invalid_field_length: 0,

‎packages/csv-parse/samples/option.info.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ assert.deepStrictEqual(records, [
99
{
1010
info: {
1111
bytes: 5,
12+
bytes_records: 5,
1213
columns: false,
1314
comment_lines: 0,
1415
empty_lines: 0,

‎packages/csv-parse/test/api.info.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ describe("API info", function () {
77
if (!info) return next(Error("Invalid assessment"));
88
info.should.eql({
99
bytes: 10,
10+
bytes_records: 16,
1011
columns: false,
1112
comment_lines: 0,
1213
empty_lines: 0,
@@ -24,6 +25,7 @@ describe("API info", function () {
2425
if (!info) return next(Error("Invalid assessment"));
2526
info.should.eql({
2627
bytes: 11,
28+
bytes_records: 17,
2729
columns: false,
2830
comment_lines: 0,
2931
empty_lines: 0,
@@ -41,6 +43,7 @@ describe("API info", function () {
4143
if (!info) return next(Error("Invalid assessment"));
4244
info.should.eql({
4345
bytes: 11,
46+
bytes_records: 11,
4447
comment_lines: 0,
4548
columns: [{ name: "a" }, { name: "b" }, { name: "c" }],
4649
empty_lines: 0,
@@ -58,6 +61,7 @@ describe("API info", function () {
5861
if (!info) return next(Error("Invalid assessment"));
5962
info.should.eql({
6063
bytes: 20,
64+
bytes_records: 41,
6165
columns: false,
6266
comment_lines: 0,
6367
empty_lines: 0,

‎packages/csv-parse/test/api.types.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ describe("API Types", function () {
8282
.sort()
8383
.should.eql([
8484
"bytes",
85+
"bytes_records",
8586
"comment_lines",
8687
"empty_lines",
8788
"invalid_field_length",
@@ -92,11 +93,10 @@ describe("API Types", function () {
9293

9394
it("Receive Callback", function (next) {
9495
parse("a\nb", function (err, records, info) {
95-
if (err !== undefined) {
96-
records!.should.eql([["a"], ["b"]]);
97-
info!.records.should.eql(2);
98-
}
99-
next(err);
96+
if (err) return next(err);
97+
records.should.eql([["a"], ["b"]]);
98+
info!.records.should.eql(2);
99+
next();
100100
});
101101
});
102102
});

‎packages/csv-parse/test/info.bytes.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import "should";
2+
import { generate } from "csv-generate";
3+
import { parse } from "../lib/index.js";
4+
5+
describe("info bytes", function () {
6+
it("info only", async function () {
7+
const parser = parse({
8+
info: true,
9+
});
10+
parser.write(Buffer.from("1,2\n"));
11+
parser.write(Buffer.from("3,4\n"));
12+
parser.write(Buffer.from("5,6\n"));
13+
parser.write(Buffer.from("7,8\n"));
14+
parser.end();
15+
const bytes = [];
16+
for await (const { info } of parser) {
17+
bytes.push(info.bytes);
18+
}
19+
bytes.should.eql([4, 8, 12, 16]);
20+
});
21+
22+
it("info with bom, columns and from_line", async function () {
23+
const parser = parse({
24+
columns: true,
25+
bom: true,
26+
from_line: 2,
27+
info: true,
28+
});
29+
parser.write(Buffer.from("\ufeffa,b\n"));
30+
parser.write(Buffer.from("1,2\n"));
31+
parser.write(Buffer.from("3,4\n"));
32+
parser.write(Buffer.from("5,6\n"));
33+
parser.write(Buffer.from("7,8\n"));
34+
parser.end();
35+
const bytes = [];
36+
for await (const { info } of parser) {
37+
bytes.push(info.bytes);
38+
}
39+
bytes.should.eql([15, 19, 23]);
40+
});
41+
42+
it("available inside fields", async function () {
43+
const parser = generate({
44+
length: 3,
45+
seed: 1,
46+
columns: 2,
47+
fixed_size: true,
48+
}).pipe(
49+
parse({
50+
cast: (value, info) => {
51+
return info.bytes;
52+
},
53+
}),
54+
);
55+
await Array.fromAsync(parser).should.finally.eql([
56+
[3, 17],
57+
[19, 27],
58+
[33, 40],
59+
]);
60+
});
61+
});

‎packages/csv-parse/test/info.bytes_records.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import "should";
2+
import { generate } from "csv-generate";
3+
import { parse } from "../lib/index.js";
4+
5+
describe("info bytes_records", function () {
6+
it("info only", async function () {
7+
const parser = parse({
8+
info: true,
9+
});
10+
parser.write(Buffer.from("1,2\n"));
11+
parser.write(Buffer.from("3,4\n"));
12+
parser.write(Buffer.from("5,6\n"));
13+
parser.write(Buffer.from("7,8\n"));
14+
parser.end();
15+
const bytes = [];
16+
for await (const { info } of parser) {
17+
bytes.push(info.bytes_records);
18+
}
19+
bytes.should.eql([4, 8, 12, 16]);
20+
});
21+
22+
it("info with bom, columns and from_line", async function () {
23+
const parser = parse({
24+
columns: true,
25+
bom: true,
26+
from_line: 2,
27+
info: true,
28+
});
29+
parser.write(Buffer.from("\ufeffa,b\n"));
30+
parser.write(Buffer.from("1,2\n"));
31+
parser.write(Buffer.from("3,4\n"));
32+
parser.write(Buffer.from("5,6\n"));
33+
parser.write(Buffer.from("7,8\n"));
34+
parser.end();
35+
const bytes = [];
36+
for await (const { info } of parser) {
37+
bytes.push(info.bytes_records);
38+
}
39+
bytes.should.eql([15, 19, 23]);
40+
});
41+
42+
it("remain constant inside fields", async function () {
43+
const parser = generate({
44+
length: 3,
45+
seed: 1,
46+
columns: 2,
47+
fixed_size: true,
48+
}).pipe(
49+
parse({
50+
cast: (value, info) => {
51+
return info.bytes_records;
52+
},
53+
}),
54+
);
55+
await Array.fromAsync(parser).should.finally.eql([
56+
[0, 0],
57+
[18, 18],
58+
[46, 46],
59+
]);
60+
});
61+
});

0 commit comments

Comments
 (0)