REM >tokenizer
PROC_tokenize("one^|uno||three^^^^|four^^^|^cuatro|", "|", "^")
END
:
DEF PROC_tokenize(src$, sep$, esc$)
LOCAL field%, char$, escaping%, i%
field% = 1
escaping% = FALSE
PRINT field%; " ";
FOR i% = 1 TO LEN src$
  char$ = MID$(src$, i%, 1)
  IF escaping% THEN
    PRINT char$;
    escaping% = FALSE
  ELSE
    CASE char$ OF
    WHEN sep$
      PRINT
      field% += 1
      PRINT field%; " ";
    WHEN esc$
      escaping% = TRUE
    OTHERWISE
      PRINT char$;
    ENDCASE
  ENDIF
NEXT
PRINT
ENDPROC