' Tokenizer: split by delimiters, quoted fields, CSV parsing
#COMPILE EXE
#DIM ALL

SUB Tokenize(BYVAL text AS STRING, BYVAL delimiters AS STRING, tok1 AS STRING, tok2 AS STRING, tok3 AS STRING, tok4 AS STRING, tok5 AS STRING, tcount AS LONG)
    LOCAL i  AS LONG
    LOCAL idx  AS LONG
    LOCAL inToken  AS LONG
    LOCAL t AS STRING
    tok1 = ""
    tok2 = ""
    tok3 = ""
    tok4 = ""
    tok5 = ""
    idx = 0
    inToken = 0
    t = ""
    FOR i = 1 TO LEN(text)
        IF INSTR(delimiters, MID$(text, i, 1)) > 0 THEN
            IF inToken THEN
                idx = idx + 1
                IF idx = 1 THEN tok1 = t
                IF idx = 2 THEN tok2 = t
                IF idx = 3 THEN tok3 = t
                IF idx = 4 THEN tok4 = t
                IF idx = 5 THEN tok5 = t
                t = ""
                inToken = 0
            END IF
        ELSE
            t = t + MID$(text, i, 1)
            inToken = 1
        END IF
    NEXT i
    IF inToken THEN
        idx = idx + 1
        IF idx = 1 THEN tok1 = t
        IF idx = 2 THEN tok2 = t
        IF idx = 3 THEN tok3 = t
        IF idx = 4 THEN tok4 = t
        IF idx = 5 THEN tok5 = t
    END IF
    tcount = idx
END SUB

SUB CSVParseLine(BYVAL line AS STRING, f1 AS STRING, f2 AS STRING, f3 AS STRING, f4 AS STRING, f5 AS STRING, fcount AS LONG)
    LOCAL i  AS LONG
    LOCAL idx  AS LONG
    LOCAL inQuote  AS LONG
    LOCAL c AS STRING
    LOCAL f AS STRING
    f1 = ""
    f2 = ""
    f3 = ""
    f4 = ""
    f5 = ""
    idx = 0
    inQuote = 0
    f = ""
    FOR i = 1 TO LEN(line)
        c = MID$(line, i, 1)
        IF c = CHR$(34) THEN
            inQuote = 1 - inQuote
        ELSEIF c = "," AND inQuote = 0 THEN
            idx = idx + 1
            IF idx = 1 THEN f1 = f
            IF idx = 2 THEN f2 = f
            IF idx = 3 THEN f3 = f
            IF idx = 4 THEN f4 = f
            IF idx = 5 THEN f5 = f
            f = ""
        ELSE
            f = f + c
        END IF
    NEXT i
    idx = idx + 1
    IF idx = 1 THEN f1 = f
    IF idx = 2 THEN f2 = f
    IF idx = 3 THEN f3 = f
    IF idx = 4 THEN f4 = f
    IF idx = 5 THEN f5 = f
    fcount = idx
END SUB

FUNCTION TokenCount(BYVAL text AS STRING, BYVAL delimiters AS STRING) AS LONG
    LOCAL i  AS LONG
    LOCAL count  AS LONG
    LOCAL inToken  AS LONG
    count = 0
    inToken = 0
    FOR i = 1 TO LEN(text)
        IF INSTR(delimiters, MID$(text, i, 1)) > 0 THEN
            IF inToken THEN
                count = count + 1
                inToken = 0
            END IF
        ELSE
            inToken = 1
        END IF
    NEXT i
    IF inToken THEN count = count + 1
    FUNCTION = count
END FUNCTION

FUNCTION PBMAIN() AS LONG
    LOCAL t1  AS STRING
    LOCAL t2  AS STRING
    LOCAL t3  AS STRING
    LOCAL t4  AS STRING
    LOCAL t5  AS STRING
    LOCAL tc AS LONG
    LOCAL f1  AS STRING
    LOCAL f2  AS STRING
    LOCAL f3  AS STRING
    LOCAL f4  AS STRING
    LOCAL f5  AS STRING
    LOCAL fc AS LONG
    PRINT "=== String Tokenizer ==="
    PRINT
    CALL Tokenize("Hello World from PBXB64 compiler", " ", t1, t2, t3, t4, t5, tc)
    PRINT "Split on space (" + FORMAT$(tc) + " tokens):"
    PRINT "  [1]: " + CHR$(34) + t1 + CHR$(34)
    PRINT "  [2]: " + CHR$(34) + t2 + CHR$(34)
    PRINT "  [3]: " + CHR$(34) + t3 + CHR$(34)
    PRINT "  [4]: " + CHR$(34) + t4 + CHR$(34)
    PRINT "  [5]: " + CHR$(34) + t5 + CHR$(34)
    PRINT
    CALL Tokenize("one,two,three,four,five", ",", t1, t2, t3, t4, t5, tc)
    PRINT "CSV split on comma (" + FORMAT$(tc) + " tokens):"
    PRINT "  [1]: " + CHR$(34) + t1 + CHR$(34)
    PRINT "  [2]: " + CHR$(34) + t2 + CHR$(34)
    PRINT "  [3]: " + CHR$(34) + t3 + CHR$(34)
    PRINT "  [4]: " + CHR$(34) + t4 + CHR$(34)
    PRINT "  [5]: " + CHR$(34) + t5 + CHR$(34)
    PRINT
    CALL CSVParseLine("Name,Age,City,State,Zip", f1, f2, f3, f4, f5, fc)
    PRINT "CSV parse (" + FORMAT$(fc) + " fields):"
    PRINT "  Field1: " + CHR$(34) + f1 + CHR$(34)
    PRINT "  Field2: " + CHR$(34) + f2 + CHR$(34)
    PRINT "  Field3: " + CHR$(34) + f3 + CHR$(34)
    PRINT
    CALL CSVParseLine(CHR$(34) + "Smith, John" + CHR$(34) + ",42," + CHR$(34) + "Portland, OR" + CHR$(34), f1, f2, f3, f4, f5, fc)
    PRINT "Quoted CSV (" + FORMAT$(fc) + " fields):"
    PRINT "  Field1: " + CHR$(34) + f1 + CHR$(34)
    PRINT "  Field2: " + CHR$(34) + f2 + CHR$(34)
    PRINT "  Field3: " + CHR$(34) + f3 + CHR$(34)
    PRINT
    PRINT "Token count test:"
    PRINT "  " + CHR$(34) + "a b c d e" + CHR$(34) + " -> " + FORMAT$(TokenCount("a b c d e", " "))
    PRINT
    PRINT "Tokenizer tests complete."
    FUNCTION = 0
END FUNCTION
