mirror of https://github.com/hwchase17/langchain
community: Add PHP language parser to document_loaders (#19850)
**Description:** Added a PHP language parser to document_loaders **Issue:** N/A **Dependencies:** N/A **Twitter handle:** N/A --------- Co-authored-by: Chester Curme <chester.curme@gmail.com>pull/9293/merge
parent
2f03bc397e
commit
e1a24d09c5
@ -0,0 +1,35 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from langchain_community.document_loaders.parsers.language.tree_sitter_segmenter import ( # noqa: E501
|
||||
TreeSitterSegmenter,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tree_sitter import Language
|
||||
|
||||
|
||||
CHUNK_QUERY = """
|
||||
[
|
||||
(function_definition) @function
|
||||
(class_declaration) @class
|
||||
(interface_declaration) @interface
|
||||
(trait_declaration) @trait
|
||||
(enum_declaration) @enum
|
||||
(namespace_definition) @namespace
|
||||
]
|
||||
""".strip()
|
||||
|
||||
|
||||
class PHPSegmenter(TreeSitterSegmenter):
|
||||
"""Code segmenter for PHP."""
|
||||
|
||||
def get_language(self) -> "Language":
|
||||
from tree_sitter_languages import get_language
|
||||
|
||||
return get_language("php")
|
||||
|
||||
def get_chunk_query(self) -> str:
|
||||
return CHUNK_QUERY
|
||||
|
||||
def make_line_comment(self, text: str) -> str:
|
||||
return f"// {text}"
|
@ -0,0 +1,68 @@
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
|
||||
from langchain_community.document_loaders.parsers.language.php import PHPSegmenter
|
||||
|
||||
|
||||
@pytest.mark.requires("tree_sitter", "tree_sitter_languages")
|
||||
class TestPHPSegmenter(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.example_code = """<?php
|
||||
namespace foo;
|
||||
|
||||
class Hello {
|
||||
public function __construct() { }
|
||||
}
|
||||
|
||||
function hello() {
|
||||
echo "Hello World!";
|
||||
}
|
||||
|
||||
interface Human {
|
||||
public function breath();
|
||||
}
|
||||
|
||||
trait Foo { }
|
||||
|
||||
enum Color
|
||||
{
|
||||
case Red;
|
||||
case Blue;
|
||||
}"""
|
||||
|
||||
self.expected_simplified_code = """<?php
|
||||
// Code for: namespace foo;
|
||||
|
||||
// Code for: class Hello {
|
||||
|
||||
// Code for: function hello() {
|
||||
|
||||
// Code for: interface Human {
|
||||
|
||||
// Code for: trait Foo { }
|
||||
|
||||
// Code for: enum Color"""
|
||||
|
||||
self.expected_extracted_code = [
|
||||
"namespace foo;",
|
||||
"class Hello {\n public function __construct() { }\n}",
|
||||
'function hello() {\n echo "Hello World!";\n}',
|
||||
"interface Human {\n public function breath();\n}",
|
||||
"trait Foo { }",
|
||||
"enum Color\n{\n case Red;\n case Blue;\n}",
|
||||
]
|
||||
|
||||
def test_is_valid(self) -> None:
|
||||
self.assertTrue(PHPSegmenter("<?php $a = 0;").is_valid())
|
||||
self.assertFalse(PHPSegmenter("<?php a ?b}+ c 1 2 3").is_valid())
|
||||
|
||||
def test_extract_functions_classes(self) -> None:
|
||||
segmenter = PHPSegmenter(self.example_code)
|
||||
extracted_code = segmenter.extract_functions_classes()
|
||||
self.assertEqual(extracted_code, self.expected_extracted_code)
|
||||
|
||||
def test_simplify_code(self) -> None:
|
||||
segmenter = PHPSegmenter(self.example_code)
|
||||
simplified_code = segmenter.simplify_code()
|
||||
self.assertEqual(simplified_code, self.expected_simplified_code)
|
Loading…
Reference in New Issue