type token =
  | Num of float
  | Var of string
  | Ident of string
  | Plus | Minus | Star | Slash | Caret
  | LParen | RParen
  | EOF

let is_digit c = c >= '0' && c <= '9'
let is_alpha c = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
let is_alphanum c = is_alpha c || is_digit c

let tokenize str =
  let len = String.length str in
  let rec aux i acc =
    if i >= len then List.rev (EOF :: acc)
    else
      match str.[i] with
      | ' ' | '\t' | '\n' -> aux (i + 1) acc
      | '+' -> aux (i + 1) (Plus :: acc)
      | '-' -> aux (i + 1) (Minus :: acc)
      | '*' -> aux (i + 1) (Star :: acc)
      | '/' -> aux (i + 1) (Slash :: acc)
      | '^' -> aux (i + 1) (Caret :: acc)
      | '(' -> aux (i + 1) (LParen :: acc)
      | ')' -> aux (i + 1) (RParen :: acc)
      | c when is_digit c || c = '.' ->
          let j = ref (i + 1) in
          while !j < len && (is_digit str.[!j] || str.[!j] = '.') do
            incr j
          done;
          let num_str = String.sub str i (!j - i) in
          aux !j (Num (float_of_string num_str) :: acc)
      | c when is_alpha c ->
          let j = ref (i + 1) in
          while !j < len && is_alphanum str.[!j] do
            incr j
          done;
          let id = String.sub str i (!j - i) in
          let tok = if !j < len && str.[!j] = '(' then Ident id else Var id in
          aux !j (tok :: acc)
      | c -> failwith (Printf.sprintf "unexpected character: %c" c)
  in
  aux 0 []