From b333b06772c89d96aacb5490d6a219fba7c09cc6 Mon Sep 17 00:00:00 2001
From: Mitja Felicijan <mitja.felicijan@gmail.com>
Date: Thu, 12 Feb 2026 20:57:17 +0100
Subject: Engage!

---
 llama.cpp/tests/peg-parser/simple-tokenize.cpp | 37 ++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 llama.cpp/tests/peg-parser/simple-tokenize.cpp

(limited to 'llama.cpp/tests/peg-parser/simple-tokenize.cpp')

diff --git a/llama.cpp/tests/peg-parser/simple-tokenize.cpp b/llama.cpp/tests/peg-parser/simple-tokenize.cpp
new file mode 100644
index 0000000..9abfa04
--- /dev/null
+++ b/llama.cpp/tests/peg-parser/simple-tokenize.cpp
@@ -0,0 +1,37 @@
+#include "simple-tokenize.h"
+
+std::vector<std::string> simple_tokenize(const std::string & input) {
+    std::vector<std::string> result;
+    std::string              current;
+
+    for (size_t i = 0; i < input.size(); i++) {
+        switch (input[i]) {
+            case ' ':
+            case '\n':
+            case '\t':
+            case '{':
+            case '}':
+            case ',':
+            case '[':
+            case '"':
+            case ']':
+            case '.':
+            case '<':
+            case '>':
+            case '=':
+            case '/':
+                if (!current.empty()) {
+                    result.push_back(current);
+                    current.clear();
+                }
+            default:;
+        }
+        current += input[i];
+    }
+
+    if (!current.empty()) {
+        result.push_back(current);
+    }
+
+    return result;
+}
-- 
cgit v1.2.3