1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123 |
x29
x29
x29
x115
x115
x115
x115
x29
x29
x29
x29
x29
x115
x115
x231
x115
x115
x186
x186
x115
x115
x115
x115
x115
x29
x29
x29
x145
x145
x145
x67206
x78309
x78309
x78309
x78309
x89373
x89373
x89373
x89373
x89373
x89373
x89373
x89373
x89373
x89373
x89373
x67206
x123164
x179119
x179119
x123164
x123167
x123167
x123164
x67206
x145
x29 |
|
// Copyright 2018-2025 the Deno authors. MIT license.
// This module is browser compatible.
import { createLPS } from "./_common.ts";
import type {
DelimiterDisposition,
DelimiterStreamOptions,
} from "./delimiter_stream.ts";
/**
* Transform a stream `string` into a stream where each chunk is divided by a
* given delimiter.
*
* If you are working with a stream of `Uint8Array`, consider using {@linkcode DelimiterStream}.
*
* If you want to split by a newline, consider using {@linkcode TextLineStream}.
*
* @example Comma-separated values
* ```ts
* import { TextDelimiterStream } from "@std/streams/text-delimiter-stream";
* import { assertEquals } from "@std/assert";
*
* const stream = ReadableStream.from([
* "alice,20,",
* ",US,",
* ]);
*
* const valueStream = stream.pipeThrough(new TextDelimiterStream(","));
*
* assertEquals(
* await Array.fromAsync(valueStream),
* ["alice", "20", "", "US", ""],
* );
* ```
*
* @example Semicolon-separated values with suffix disposition
* ```ts
* import { TextDelimiterStream } from "@std/streams/text-delimiter-stream";
* import { assertEquals } from "@std/assert";
*
* const stream = ReadableStream.from([
* "const a = 42;;let b =",
* " true;",
* ]);
*
* const valueStream = stream.pipeThrough(
* new TextDelimiterStream(";", { disposition: "suffix" }),
* );
*
* assertEquals(
* await Array.fromAsync(valueStream),
* ["const a = 42;", ";", "let b = true;", ""],
* );
* ```
*/
export class TextDelimiterStream extends TransformStream<string, string> {
#buf = "";
#delimiter: string;
#inspectIndex = 0;
#matchIndex = 0;
#delimLPS: Uint8Array;
#disp: DelimiterDisposition;
/**
* Constructs a new instance.
*
* @param delimiter A delimiter to split the stream by.
* @param options Options for the stream.
*/
constructor(
delimiter: string,
options?: DelimiterStreamOptions,
) {
super({
transform: (chunk, controller) => {
this.#handle(chunk, controller);
},
flush: (controller) => {
controller.enqueue(this.#buf);
},
});
this.#delimiter = delimiter;
this.#delimLPS = createLPS(new TextEncoder().encode(delimiter));
this.#disp = options?.disposition ?? "discard";
}
#handle(
chunk: string,
controller: TransformStreamDefaultController<string>,
) {
this.#buf += chunk;
let localIndex = 0;
while (this.#inspectIndex < this.#buf.length) {
if (chunk[localIndex] === this.#delimiter[this.#matchIndex]) {
this.#inspectIndex++;
localIndex++;
this.#matchIndex++;
if (this.#matchIndex === this.#delimiter.length) {
// Full match
const start = this.#inspectIndex - this.#delimiter.length;
const end = this.#disp === "suffix" ? this.#inspectIndex : start;
const copy = this.#buf.slice(0, end);
controller.enqueue(copy);
const shift = this.#disp === "prefix" ? start : this.#inspectIndex;
this.#buf = this.#buf.slice(shift);
this.#inspectIndex = this.#disp === "prefix"
? this.#delimiter.length
: 0;
this.#matchIndex = 0;
}
} else {
if (this.#matchIndex === 0) {
this.#inspectIndex++;
localIndex++;
} else {
this.#matchIndex = this.#delimLPS[this.#matchIndex - 1]!;
}
}
}
}
}
|