diff --git a/providers/utils/heuristic_tool_parser.py b/providers/utils/heuristic_tool_parser.py index 9e0c03ac..01c91b0b 100644 --- a/providers/utils/heuristic_tool_parser.py +++ b/providers/utils/heuristic_tool_parser.py @@ -99,6 +99,16 @@ class HeuristicToolParser: while True: param_match = self.param_pattern.search(self.buffer) if param_match and "" in param_match.group(0): + # Detect any content before the parameter match and preserve it + pre_match_text = self.buffer[: param_match.start()] + if pre_match_text.strip(): + # If there's non-whitespace text, we should probably treat it as content + # However, purely whitespace might be formatting + filtered_output += pre_match_text + elif pre_match_text: + # Preserve whitespace too just in case + filtered_output += pre_match_text + key = param_match.group(1).strip() val = param_match.group(2).strip() self.current_parameters[key] = val @@ -113,6 +123,11 @@ class HeuristicToolParser: if "●" in self.buffer: # Next tool call starting or something else, close current + # But first, capture any text before the ● + idx = self.buffer.find("●") + if idx > 0: + filtered_output += self.buffer[:idx] + self.buffer = self.buffer[idx:] finished_tool_call = True elif ( len(self.buffer) > 0 @@ -122,6 +137,11 @@ class HeuristicToolParser: # We have text that doesn't look like a tag, and we already parsed some or are in param state # Let's see if we have trailing param starts if "" + # Split at various points + full_text = "● val" + + for i in range(len(full_text)): + p = HeuristicToolParser() + chunk1 = full_text[:i] + chunk2 = full_text[i:] + + tools = [] + filtered, t = p.feed(chunk1) + tools.extend(t) + filtered2, t = p.feed(chunk2) + tools.extend(t) + tools.extend(p.flush()) + + if len(tools) != 1: + print(f"Failed split at index {i}: '{chunk1}' | '{chunk2}'") + + assert len(tools) == 1, f"Failed split at index {i}" + assert tools[0]["name"] == "Test" + assert tools[0]["input"] == {"arg": "val"} + + +def test_value_with_special_chars(): + parser = HeuristicToolParser() + # Value with > inside + text = "● a > b" + _, tools = parser.feed(text) + tools.extend(parser.flush()) + + assert len(tools) == 1 + assert tools[0]["input"]["arg"] == "a > b" + + +def test_multiple_params_split(): + full_text = ( + "● v1v2" + ) + + for i in range(len(full_text)): + p = HeuristicToolParser() + tools = [] + _, t = p.feed(full_text[:i]) + tools.extend(t) + _, t = p.feed(full_text[i:]) + tools.extend(t) + tools.extend(p.flush()) + + assert len(tools) == 1, f"Failed split at {i}" + assert tools[0]["input"] == {"p1": "v1", "p2": "v2"} + + +def test_incomplete_tag_flush(): + p = HeuristicToolParser() + p.feed("● hello") + tools = p.flush() + + assert len(tools) == 1 + assert tools[0]["input"]["msg"] == "hello" + + +def test_garbage_interleaved(): + p = HeuristicToolParser() + tools = [] + _, t = p.feed("Some text ") + tools.extend(t) + _, t = p.feed("● 1") + tools.extend(t) + _, t = p.feed(" more text ") + tools.extend(t) + _, t = p.feed("● 2") + tools.extend(t) + tools.extend(p.flush()) + + assert len(tools) == 2 + assert tools[0]["name"] == "T1" + assert tools[1]["name"] == "T2" + + +def test_text_between_params_lost(): + p = HeuristicToolParser() + # " text1 " is between function end and first param + # " text2 " is between params + text = "● text1 1 text2 2" + filtered, tools = p.feed(text) + tools.extend(p.flush()) + + # Check if "text1" and "text2" are preserved in filtered output + assert "text1" in filtered + assert "text2" in filtered + assert tools[0]["input"] == {"a": "1", "b": "2"}