Contoh memproses hasil OCR menjadi struktur terorganisir:

image = Image.open("mejakerja.jpg").convert("RGB")

Segmentasi berdasarkan deskripsi
result = runflorence(

    model, processor, image,
    "EXPRESSIONSEGMENTATION>",

    textinput="the laptop on the left side of the desk"
)

seg = result["EXPRESSIONSEGMENTATION>"]
print("Polygons segmentasi:", seg["polygons"])
print("Label:", seg["labels"])

Membuat mask dari polygon hasil segmentasi:

import json
from torch.utils.data import Dataset, DataLoader
from transformers import (
    AutoProcessor, AutoModelForCausalLM,
    TrainingArguments, Trainer
)

class Florence2Dataset(Dataset):
    """Dataset kustom untuk fine-tuning Florence-2."""

    def init(self, datadir, annotationsfile, processor, taskprompt):
        self.datadir = Path(datadir)
        self.processor = processor
        self.taskprompt = taskprompt

        with open(annotationsfile, "r") as f:

            self.annotations = json.load(f)

    def len(self):
        return len(self.annotations)

    def getitem(self, idx):
        ann = self.annotations[idx]

        # Muat gambar
        image = Image.open(
            self.datadir / ann["image"]
        ).convert("RGB")

        # Siapkan input dan target
        # Format target tergantung task:
        # - Captioning: teks biasa
        # - Detection: format khusus Florence-2
        prompt = self.taskprompt

        target = ann["target"]  # Teks target sesuai format task

        # Proses input
        inputs = self.processor(
            text=prompt,
            images=image,
            returntensors="pt",
            padding="maxlength",

            maxlength=512,
            truncation=True
        )

        # Proses target/labels
        labels = self.processor.tokenizer(
            target,
            returntensors="pt",

            padding="maxlength",
            maxlength=512,

            truncation=True
        )

        return {
            "inputids": inputs["inputids"].squeeze(),

            "pixelvalues": inputs["pixelvalues"].squeeze(),

            "attentionmask": inputs["attentionmask"].squeeze(),

            "labels": labels["inputids"].squeeze()
        }

Format anotasi untuk berbagai tugas:

# Contoh file annotations.json untuk captioning
annotationscaption = [

    {
        "image": "img001.jpg",
        "target": "A red excavator digging on a construction site"
    },
    {
        "image": "img002.jpg",

        "target": "Workers wearing safety helmets inspecting equipment"
    }
]

Contoh untuk object detection
Target menggunakan format Florence-2: "x1>y1>x2>y2>label"
annotationsdetection = [
    {
        "image": "img001.jpg",

        "target": "excavator120>200>450>500>"
                  "worker50>100>150>400>"
    }
]

Simpan anotasi
with open("annotations.json", "w") as f:
    json.dump(annotationscaption, f, indent=2)

Menjalankan fine-tuning:

# Muat model untuk fine-tuning
MODELID = "microsoft/Florence-2-base-ft"

processor = AutoProcessor.frompretrained(MODELID, trustremotecode=True)

model = AutoModelForCausalLM.frompretrained(
    MODELID,

    trustremotecode=True,

    torchdtype=torch.float16
)

Siapkan dataset
traindataset = Florence2Dataset(

    datadir="dataset/train/images",
    annotationsfile="dataset/train/annotations.json",

    processor=processor,
    taskprompt=""
)

valdataset = Florence2Dataset(

    datadir="dataset/val/images",
    annotationsfile="dataset/val/annotations.json",

    processor=processor,
    taskprompt=""
)

Konfigurasi training
trainingargs = TrainingArguments(

    outputdir="florence2-finetuned",
    numtrainepochs=10,
    perdevicetrainbatchsize=4,
    perdeviceevalbatchsize=4,
    gradientaccumulationsteps=4,
    learningrate=1e-5,

    weightdecay=0.01,
    warmupratio=0.1,

    lrschedulertype="cosine",

    loggingsteps=50,
    evalstrategy="steps",

    evalsteps=200,
    savesteps=200,

    savetotallimit=3,

    loadbestmodelatend=True,

    metricforbestmodel="evalloss",

    fp16=True,
    dataloadernumworkers=4,

    removeunusedcolumns=False,

    reportto="none",
)

Buat trainer dan mulai training
trainer = Trainer(
    model=model,
    args=trainingargs,

    traindataset=traindataset,

    evaldataset=valdataset,

)

trainer.train()

Simpan model
trainer.savemodel("florence2-finetuned/best")
processor.savepretrained("florence2-finetuned/best")

# Muat model fine-tuned
finetunedmodel = AutoModelForCausalLM.frompretrained(

    "florence2-finetuned/best",
    trustremotecode=True,

    torchdtype=torch.float16,
    devicemap="auto"

)
finetunedprocessor = AutoProcessor.frompretrained(

    "florence2-finetuned/best",
    trustremotecode=True

)

Gunakan seperti biasa
result = runflorence(
    finetunedmodel, finetunedprocessor,
    image, ""
)
print(result)

Integrasi dengan Supervision untuk Visualisasi

Library supervision dari Roboflow menyediakan tools visualisasi yang sangat baik untuk hasil deteksi:

import supervision as sv
import numpy as np
from PIL import Image

def florencetosvdetections(result, taskkey, imagesize):

    """Konversi hasil Florence-2 ke format supervision Detections."""
    data = result[taskkey]

    if "bboxes" not in data or len(data["bboxes"]) == 0:
        return sv.Detections.empty()

    bboxes = np.array(data["bboxes"])
    labels = data.get("labels", [])

    return sv.Detections(
        xyxy=bboxes,
        data={"classname": np.array(labels)}

    )

Jalankan deteksi
image = Image.open("foto.jpg").convert("RGB")
result = runflorence(model, processor, image, "")

Konversi ke format supervision
detections = florencetosvdetections(

    result, "", (image.width, image.height)
)

Visualisasi dengan bounding boxes
imagenp = np.array(image)

bboxannotator = sv.BoxAnnotator(

    thickness=2,
    color=sv.ColorPalette.DEFAULT
)
labelannotator = sv.LabelAnnotator(
    textscale=0.5,

    textpadding=5,
    color=sv.ColorPalette.DEFAULT
)

annotated = bboxannotator.annotate(

    scene=imagenp.copy(),
    detections=detections
)
annotated = labelannotator.annotate(

    scene=annotated,
    detections=detections,
    labels=detections.data["classname"].tolist()
)

Simpan hasil
sv.plotimage(annotated, size=(12, 8))

Image.fromarray(annotated).save("deteksihasil.jpg")
print(f"Terdeteksi {len(detections)} objek")

Visualisasi untuk dense region captioning:

# Dense region captioning dengan visualisasi
result = runflorence(model, processor, image, "REGIONCAPTION>")


detections = florencetosvdetections(
    result, "REGIONCAPTION>", (image.width, image.height)
)

Annotasi dengan caption per region
annotated = bboxannotator.annotate(

    scene=np.array(image).copy(),
    detections=detections
)
annotated = labelannotator.annotate(
    scene=annotated,
    detections=detections,
    labels=detections.data["classname"].tolist()

)

Image.fromarray(annotated).save("densecaptionhasil.jpg")

def visualizeocrresults(image, ocrresult):
    """Visualisasi hasil OCR dengan supervision."""
    data = ocrresult["WITHREGION>"]


    if "quadboxes" not in data or len(data["quadboxes"]) == 0:

        return np.array(image)

    # Konversi quadboxes ke xyxy format
    bboxes = []
    for quad in data["quadboxes"]:

        xcoords = [quad[i] for i in range(0, 8, 2)]
        ycoords = [quad[i] for i in range(1, 8, 2)]

        bboxes.append([
            min(xcoords), min(ycoords),

            max(xcoords), max(ycoords)

        ])

    detections = sv.Detections(
        xyxy=np.array(bboxes),
        data={"classname": np.array(data["labels"])}
    )

    annotated = bboxannotator.annotate(

        scene=np.array(image).copy(),
        detections=detections
    )
    annotated = labelannotator.annotate(
        scene=annotated,
        detections=detections,
        labels=data["labels"]
    )

    return annotated

result = runflorence(model, processor, image, "WITHREGION>")

ocrvis = visualizeocrresults(image, result)
Image.fromarray(ocrvis).save("ocrhasil.jpg")

import json
from pathlib import Path
from dataclasses import dataclass, field, asdict
from PIL import Image
import numpy as np

@dataclass
class DocumentElement:
    """Representasi elemen dalam dokumen."""
    type: str           # "text", "table", "figure", "header"
    bbox: list          # [x1, y1, x2, y2]
    content: str = ""   # Teks atau caption
    confidence: float = 0.0

@dataclass
class DocumentAnalysis:
    """Hasil analisis dokumen lengkap."""
    filename: str
    fulltext: str = ""

    caption: str = ""
    detailedcaption: str = ""
    elements: list = field(defaultfactory=list)


    def todict(self):
        return asdict(self)

class DocumentAnalyzer:
    """Pipeline analisis dokumen menggunakan Florence-2."""

    def init(self, modelid="microsoft/Florence-2-large-ft"):

        self.processor = AutoProcessor.frompretrained(
            modelid, trustremotecode=True

        )
        self.model = AutoModelForCausalLM.frompretrained(
            modelid,

            trustremotecode=True,

            torchdtype=torch.float16,
            devicemap="auto"

        )

    def run(self, image, task, textinput=None):

        """Helper untuk menjalankan inferensi."""
        return runflorence(
            self.model, self.processor, image, task, textinput

        )

    def analyze(self, imagepath):
        """Analisis dokumen lengkap."""
        image = Image.open(imagepath).convert("RGB")

        doc = DocumentAnalysis(filename=str(imagepath))

        # Langkah 1: Dapatkan caption keseluruhan
        captionresult = self.run(image, "")
        doc.caption = captionresult[""]


        detailresult = self.run(image, "CAPTION>")

        doc.detailedcaption = detailresult["CAPTION>"]

        # Langkah 2: Ekstrak teks dengan OCR
        ocrresult = self.run(image, "WITHREGION>")

        ocrdata = ocrresult["WITHREGION>"]


        textelements = []
        if "quadboxes" in ocrdata:
            for quad, text in zip(
                ocrdata["quadboxes"], ocrdata["labels"]

            ):
                xcoords = [quad[i] for i in range(0, 8, 2)]
                ycoords = [quad[i] for i in range(1, 8, 2)]

                bbox = [
                    min(xcoords), min(ycoords),

                    max(xcoords), max(ycoords)

                ]
                textelements.append(
                    DocumentElement("text", bbox, text)
                )

        # Urutkan teks dari atas ke bawah
        textelements.sort(

            key=lambda e: (
                round(e.bbox[1] / 30)  30, e.bbox[0]
            )
        )
        doc.elements.extend(textelements)
        doc.fulltext = " ".join(e.content for e in textelements)

        # Langkah 3: Deteksi elemen visual (gambar, tabel, dll)
        odresult = self.run(image, "")
        oddata = odresult[""]

        if "bboxes" in oddata:

            for bbox, label in zip(
                oddata["bboxes"], oddata["labels"]

            ):
                # Identifikasi tipe elemen
                labellower = label.lower()
                if any(w in labellower for w in

                       ["table", "chart", "graph"]):
                    elemtype = "table"
                elif any(w in labellower for w in

                         ["image", "photo", "figure", "picture"]):
                    elemtype = "figure"
                else:
                    elemtype = "object"


                # Dapatkan deskripsi untuk region ini
                x1, y1, x2, y2 = bbox
                nx1 = int(x1 / image.width  999)

                ny1 = int(y1 / image.height  999)
                nx2 = int(x2 / image.width  999)

                ny2 = int(y2 / image.height  999)
                region = (f"{nx1}>{ny1}>"
                         f"{nx2}>{ny2}>")

                descresult = self.run(

                    image, "TODESCRIPTION>",

                    textinput=region
                )
                description = descresult.get(

                    "TODESCRIPTION>", label

                )

                doc.elements.append(
                    DocumentElement(
                        elemtype, list(bbox),
                        str(description)
                    )
                )

        return doc

    def analyzebatch(self, imagedir, outputfile):

        """Analisis batch dokumen dan simpan hasil."""
        imagedir = Path(imagedir)

        extensions = {".jpg", ".jpeg", ".png", ".bmp", ".tiff"}
        images = [
            p for p in imagedir.iterdir()
            if p.suffix.lower() in extensions
        ]

        results = []
        for i, imgpath in enumerate(images):

            print(f"\nMemproses [{i+1}/{len(images)}]: {imgpath.name}")
            try:
                doc = self.analyze(imgpath)

                results.append(doc.todict())
                print(f"  Caption: {doc.caption}")
                print(f"  Elemen: {len(doc.elements)}")
                print(f"  Teks: {doc.fulltext[:100]}...")

            except Exception as e:
                print(f"  Error: {e}")
                results.append({
                    "filename": str(imgpath),
                    "error": str(e)
                })

        # Simpan hasil
        with open(outputfile, "w", encoding="utf-8") as f:

            json.dump(results, f, indent=2, ensureascii=False)

        print(f"\nHasil disimpan ke {outputfile}")

        return results

Penggunaan
analyzer = DocumentAnalyzer("microsoft/Florence-2-large-ft")

Analisis satu dokumen
doc = analyzer.analyze("dokumeninvoice.jpg")
print(f"Caption: {doc.caption}")
print(f"Teks: {doc.fulltext}")

print(f"Jumlah elemen: {len(doc.elements)}")

Analisis batch
results = analyzer.analyzebatch(
    "folderdokumen/",

    "hasilanalisis.json"
)

# Gunakan base-ft untuk kecepatan dan efisiensi memori
Gunakan large-ft untuk akurasi terbaik

Perbandingan varian:
Florence-2-base     : 232M params, inferensi cepat, cocok untuk edge
Florence-2-base-ft  : 232M params + fine-tuned, keseimbangan terbaik
Florence-2-large    : 771M params, akurasi tinggi
Florence-2-large-ft : 771M params + fine-tuned, performa terbaik