feat: Extracting picture data for raster images found in PPTX (#349)

* Added picture data for pptx pictures

Signed-off-by: Maksym Lysak <mly@zurich.ibm.com>

* Added tests for pptx

Signed-off-by: Maksym Lysak <mly@zurich.ibm.com>

* Inferring image DPI from pptx file

Signed-off-by: Maksym Lysak <mly@zurich.ibm.com>

---------

Signed-off-by: Maksym Lysak <mly@zurich.ibm.com>
Co-authored-by: Maksym Lysak <mly@zurich.ibm.com>

docling/backend/mspowerpoint_backend.py

@@ -10,11 +10,13 @@ from docling_core.types.doc import (
     DoclingDocument,
     DocumentOrigin,
     GroupLabel,
+    ImageRef,
     ProvenanceItem,
     Size,
     TableCell,
     TableData,
 )
+from PIL import Image
 from pptx import Presentation
 from pptx.enum.shapes import MSO_SHAPE_TYPE, PP_PLACEHOLDER
 
@@ -268,9 +270,22 @@ class MsPowerpointDocumentBackend(DeclarativeDocumentBackend, PaginatedDocumentB
         return
 
     def handle_pictures(self, shape, parent_slide, slide_ind, doc):
+        # Get the image bytes
+        image = shape.image
+        image_bytes = image.blob
+        im_dpi, _ = image.dpi
+
+        # Open it with PIL
+        pil_image = Image.open(BytesIO(image_bytes))
+
         # shape has picture
         prov = self.generate_prov(shape, slide_ind, "")
-        doc.add_picture(parent=parent_slide, caption=None, prov=prov)
+        doc.add_picture(
+            parent=parent_slide,
+            image=ImageRef.from_pil(image=pil_image, dpi=im_dpi),
+            caption=None,
+            prov=prov,
+        )
         return
 
     def handle_tables(self, shape, parent_slide, slide_ind, doc):

tests/data/groundtruth/docling_v2/powerpoint_sample.pptx.itxt

@@ -0,0 +1,35 @@
+item-0 at level 0: unspecified: group _root_
+  item-1 at level 1: chapter: group slide-0
+    item-2 at level 2: title: Test Table Slide
+    item-3 at level 2: paragraph: With footnote
+    item-4 at level 2: table with [9x7]
+  item-5 at level 1: chapter: group slide-1
+    item-6 at level 2: title: Second slide title
+    item-7 at level 2: paragraph: Let’s introduce a list
+    item-8 at level 2: paragraph: With foo
+    item-9 at level 2: paragraph: Bar
+    item-10 at level 2: paragraph: And baz things
+    item-11 at level 2: paragraph: A rectangle shape with this text inside.
+  item-12 at level 1: chapter: group slide-2
+    item-13 at level 2: ordered_list: group list
+      item-14 at level 3: list_item: List item4
+      item-15 at level 3: list_item: List item5
+      item-16 at level 3: list_item: List item6
+    item-17 at level 2: list: group list
+      item-18 at level 3: list_item: I1
+      item-19 at level 3: list_item: I2
+      item-20 at level 3: list_item: I3
+      item-21 at level 3: list_item: I4
+    item-22 at level 2: paragraph: Some info:
+    item-23 at level 2: list: group list
+      item-24 at level 3: list_item: Item A
+      item-25 at level 3: list_item: Item B
+    item-26 at level 2: paragraph: Maybe a list?
+    item-27 at level 2: ordered_list: group list
+      item-28 at level 3: list_item: List1
+      item-29 at level 3: list_item: List2
+      item-30 at level 3: list_item: List3
+    item-31 at level 2: list: group list
+      item-32 at level 3: list_item: l1 
+      item-33 at level 3: list_item: l2
+      item-34 at level 3: list_item: l3

tests/data/groundtruth/docling_v2/powerpoint_sample.pptx.md

@@ -0,0 +1,50 @@
+# Test Table Slide
+
+With footnote
+
+|    | Class1          | Class1          | Class1   | Class2   | Class2   | Class2   |
+|----|-----------------|-----------------|----------|----------|----------|----------|
+|    | A merged with B | A merged with B | C        | A        | B        | C        |
+| R1 | True            | False           |          | False    | True     | True     |
+| R2 |                 |                 | True     | False    |          |          |
+| R3 | False           |                 |          |          | False    |          |
+| R3 |                 | True            |          | True     |          |          |
+| R4 |                 |                 | False    |          | False    |          |
+| R4 |                 | True            |          | True     | False    | False    |
+| R4 | True            | False           | True     | False    | True     | False    |
+
+# Second slide title
+
+Let’s introduce a list
+
+With foo
+
+Bar
+
+And baz things
+
+A rectangle shape with this text inside.
+
+1. List item4
+2. List item5
+3. List item6
+
+- I1
+- I2
+- I3
+- I4
+
+Some info:
+
+- Item A
+- Item B
+
+Maybe a list?
+
+1. List1
+2. List2
+3. List3
+
+- l1 
+- l2
+- l3

tests/data/groundtruth/docling_v2/powerpoint_with_image.pptx.itxt

@@ -0,0 +1,5 @@
+item-0 at level 0: unspecified: group _root_
+  item-1 at level 1: chapter: group slide-0
+    item-2 at level 2: title: Docling
+    item-3 at level 2: paragraph: Image test
+    item-4 at level 2: picture

tests/data/groundtruth/docling_v2/powerpoint_with_image.pptx.md

@@ -0,0 +1,5 @@
+# Docling
+
+Image test
+
+<!-- image -->

tests/test_backend_pptx.py

@@ -0,0 +1,72 @@
+import json
+import os
+from pathlib import Path
+
+from docling.datamodel.base_models import InputFormat
+from docling.datamodel.document import ConversionResult
+from docling.document_converter import DocumentConverter
+
+GENERATE = False
+
+
+def get_pptx_paths():
+
+    # Define the directory you want to search
+    directory = Path("./tests/data/pptx/")
+
+    # List all PPTX files in the directory and its subdirectories
+    pptx_files = sorted(directory.rglob("*.pptx"))
+    return pptx_files
+
+
+def get_converter():
+
+    converter = DocumentConverter(allowed_formats=[InputFormat.PPTX])
+
+    return converter
+
+
+def verify_export(pred_text: str, gtfile: str):
+
+    if not os.path.exists(gtfile) or GENERATE:
+        with open(gtfile, "w") as fw:
+            fw.write(pred_text)
+
+        return True
+
+    else:
+        with open(gtfile, "r") as fr:
+            true_text = fr.read()
+
+        assert pred_text == true_text, "pred_itxt==true_itxt"
+        return pred_text == true_text
+
+
+def test_e2e_pptx_conversions():
+
+    pptx_paths = get_pptx_paths()
+    converter = get_converter()
+
+    for pptx_path in pptx_paths:
+        # print(f"converting {pptx_path}")
+
+        gt_path = (
+            pptx_path.parent.parent / "groundtruth" / "docling_v2" / pptx_path.name
+        )
+
+        conv_result: ConversionResult = converter.convert(pptx_path)
+
+        doc: DoclingDocument = conv_result.document
+
+        pred_md: str = doc.export_to_markdown()
+        assert verify_export(pred_md, str(gt_path) + ".md"), "export to md"
+
+        pred_itxt: str = doc._export_to_indented_text(
+            max_text_len=70, explicit_tables=False
+        )
+        assert verify_export(
+            pred_itxt, str(gt_path) + ".itxt"
+        ), "export to indented-text"
+
+        pred_json: str = json.dumps(doc.export_to_dict(), indent=2)
+        assert verify_export(pred_json, str(gt_path) + ".json"), "export to json"