fix: parse html with omitted body tag (#818)
* fix: parse HTML files without body tag Parse HTML files without 'body' tag, since it is optional in HTML5 specification. Signed-off-by: Cesar Berrospi Ramis <75900930+ceberam@users.noreply.github.com> * test: ensure docling converts HTML without body tag Signed-off-by: Cesar Berrospi Ramis <75900930+ceberam@users.noreply.github.com> --------- Signed-off-by: Cesar Berrospi Ramis <75900930+ceberam@users.noreply.github.com>
This commit is contained in:
Cesar Berrospi Ramis
GitHub
parent
95b293a723
commit
a112d7a035
@ -78,10 +78,11 @@ class HTMLDocumentBackend(DeclarativeDocumentBackend):
|
||||
|
||||
if self.is_valid():
|
||||
assert self.soup is not None
|
||||
content = self.soup.body or self.soup
|
||||
# Replace <br> tags with newline characters
|
||||
for br in self.soup.body.find_all("br"):
|
||||
for br in content.find_all("br"):
|
||||
br.replace_with("\n")
|
||||
doc = self.walk(self.soup.body, doc)
|
||||
doc = self.walk(content, doc)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Cannot convert doc with {self.document_hash} because the backend failed to init."
|
||||
|
||||
3
tests/data/groundtruth/docling_v2/example_05.html.itxt
Normal file
3
tests/data/groundtruth/docling_v2/example_05.html.itxt
Normal file
@ -0,0 +1,3 @@
|
||||
item-0 at level 0: unspecified: group _root_
|
||||
item-1 at level 1: title: Omitted html and body tags
|
||||
item-2 at level 2: table with [4x3]
|
||||
329
tests/data/groundtruth/docling_v2/example_05.html.json
Normal file
329
tests/data/groundtruth/docling_v2/example_05.html.json
Normal file
@ -0,0 +1,329 @@
|
||||
{
|
||||
"schema_name": "DoclingDocument",
|
||||
"version": "1.0.0",
|
||||
"name": "example_05",
|
||||
"origin": {
|
||||
"mimetype": "text/html",
|
||||
"binary_hash": 1499806583410518209,
|
||||
"filename": "example_05.html"
|
||||
},
|
||||
"furniture": {
|
||||
"self_ref": "#/furniture",
|
||||
"children": [],
|
||||
"name": "_root_",
|
||||
"label": "unspecified"
|
||||
},
|
||||
"body": {
|
||||
"self_ref": "#/body",
|
||||
"children": [
|
||||
{
|
||||
"$ref": "#/texts/0"
|
||||
}
|
||||
],
|
||||
"name": "_root_",
|
||||
"label": "unspecified"
|
||||
},
|
||||
"groups": [],
|
||||
"texts": [
|
||||
{
|
||||
"self_ref": "#/texts/0",
|
||||
"parent": {
|
||||
"$ref": "#/body"
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"$ref": "#/tables/0"
|
||||
}
|
||||
],
|
||||
"label": "title",
|
||||
"prov": [],
|
||||
"orig": "Omitted html and body tags",
|
||||
"text": "Omitted html and body tags"
|
||||
}
|
||||
],
|
||||
"pictures": [],
|
||||
"tables": [
|
||||
{
|
||||
"self_ref": "#/tables/0",
|
||||
"parent": {
|
||||
"$ref": "#/texts/0"
|
||||
},
|
||||
"children": [],
|
||||
"label": "table",
|
||||
"prov": [],
|
||||
"captions": [],
|
||||
"references": [],
|
||||
"footnotes": [],
|
||||
"data": {
|
||||
"table_cells": [
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 0,
|
||||
"end_row_offset_idx": 1,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Header 1",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 0,
|
||||
"end_row_offset_idx": 1,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Header 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 2,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Row 1 & 2, Col 1 (rowspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 2,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 2,
|
||||
"text": "Row 1, Col 2",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 2,
|
||||
"start_col_offset_idx": 2,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 1, Col 3",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 2,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 2, Col 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Row 3, Col 1",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 2,
|
||||
"text": "Row 3, Col 2",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 2,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 3, Col 3",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
}
|
||||
],
|
||||
"num_rows": 4,
|
||||
"num_cols": 3,
|
||||
"grid": [
|
||||
[
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 0,
|
||||
"end_row_offset_idx": 1,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Header 1",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 0,
|
||||
"end_row_offset_idx": 1,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Header 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 0,
|
||||
"end_row_offset_idx": 1,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Header 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"row_span": 2,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Row 1 & 2, Col 1 (rowspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 2,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 2,
|
||||
"text": "Row 1, Col 2",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 2,
|
||||
"start_col_offset_idx": 2,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 1, Col 3",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"row_span": 2,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 1,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Row 1 & 2, Col 1 (rowspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 2,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 2, Col 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 2,
|
||||
"start_row_offset_idx": 2,
|
||||
"end_row_offset_idx": 3,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 2, Col 2 & 3 (colspan)",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 0,
|
||||
"end_col_offset_idx": 1,
|
||||
"text": "Row 3, Col 1",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 1,
|
||||
"end_col_offset_idx": 2,
|
||||
"text": "Row 3, Col 2",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
},
|
||||
{
|
||||
"row_span": 1,
|
||||
"col_span": 1,
|
||||
"start_row_offset_idx": 3,
|
||||
"end_row_offset_idx": 4,
|
||||
"start_col_offset_idx": 2,
|
||||
"end_col_offset_idx": 3,
|
||||
"text": "Row 3, Col 3",
|
||||
"column_header": false,
|
||||
"row_header": false,
|
||||
"row_section": false
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"key_value_items": [],
|
||||
"pages": {}
|
||||
}
|
||||
7
tests/data/groundtruth/docling_v2/example_05.html.md
Normal file
7
tests/data/groundtruth/docling_v2/example_05.html.md
Normal file
@ -0,0 +1,7 @@
|
||||
# Omitted html and body tags
|
||||
|
||||
| Header 1 | Header 2 & 3 (colspan) | Header 2 & 3 (colspan) |
|
||||
|----------------------------|----------------------------|----------------------------|
|
||||
| Row 1 & 2, Col 1 (rowspan) | Row 1, Col 2 | Row 1, Col 3 |
|
||||
| Row 1 & 2, Col 1 (rowspan) | Row 2, Col 2 & 3 (colspan) | Row 2, Col 2 & 3 (colspan) |
|
||||
| Row 3, Col 1 | Row 3, Col 2 | Row 3, Col 3 |
|
||||
20
tests/data/html/example_05.html
Normal file
20
tests/data/html/example_05.html
Normal file
@ -0,0 +1,20 @@
|
||||
<h1>Omitted html and body tags</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Header 1</th>
|
||||
<th colspan="2">Header 2 & 3 (colspan)</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td rowspan="2">Row 1 & 2, Col 1 (rowspan)</td>
|
||||
<td>Row 1, Col 2</td>
|
||||
<td>Row 1, Col 3</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="2">Row 2, Col 2 & 3 (colspan)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Row 3, Col 1</td>
|
||||
<td>Row 3, Col 2</td>
|
||||
<td>Row 3, Col 3</td>
|
||||
</tr>
|
||||
</table>
|
||||
@ -6,6 +6,7 @@ from docling.backend.html_backend import HTMLDocumentBackend
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.document import (
|
||||
ConversionResult,
|
||||
DoclingDocument,
|
||||
InputDocument,
|
||||
SectionHeaderItem,
|
||||
)
|
||||
@ -44,7 +45,7 @@ def get_html_paths():
|
||||
# Define the directory you want to search
|
||||
directory = Path("./tests/data/html/")
|
||||
|
||||
# List all PDF files in the directory and its subdirectories
|
||||
# List all HTML files in the directory and its subdirectories
|
||||
html_files = sorted(directory.rglob("*.html"))
|
||||
return html_files
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user