19 lines
397 B
Plaintext
19 lines
397 B
Plaintext
MODULE Tokenize EXPORTS Main;
|
|
|
|
IMPORT IO, TextConv;
|
|
|
|
TYPE Texts = REF ARRAY OF TEXT;
|
|
|
|
VAR tokens: Texts;
|
|
string := "Hello,How,Are,You,Today";
|
|
sep := SET OF CHAR {','};
|
|
|
|
BEGIN
|
|
tokens := NEW(Texts, TextConv.ExplodedSize(string, sep));
|
|
TextConv.Explode(string, tokens^, sep);
|
|
FOR i := FIRST(tokens^) TO LAST(tokens^) DO
|
|
IO.Put(tokens[i] & ".");
|
|
END;
|
|
IO.Put("\n");
|
|
END Tokenize.
|