init

2026-06-22 08:31:19 +08:00
commit 83dfe18977
14 changed files with 534 additions and 0 deletions
@@ -0,0 +1,13 @@
+.venv/
+venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.env
+data/
+models/
+openocr_output/
+dist/
+build/
+.git/
@@ -0,0 +1,8 @@
+OPENOCR_OUTPUT_DIR=data/output
+OPENOCR_UPLOAD_DIR=data/uploads
+OPENOCR_USE_GPU=false
+OPENOCR_AUTO_DOWNLOAD=true
+OPENOCR_LAYOUT_THRESHOLD=0.4
+OPENOCR_MAX_LENGTH=2048
+OPENOCR_CORS_ORIGINS=*
+OPENOCR_LOG_LEVEL=info
@@ -0,0 +1,61 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.egg
+*.egg-info/
+.eggs/
+dist/
+build/
+*.manifest
+*.spec
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# Test / lint / type check
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# Environment and secrets
+.env
+.env.*
+!.env.example
+
+# Runtime data and model cache
+data/
+models/
+openocr_output/
+.cache/
+.huggingface/
+.modelscope/
+
+# Large model weights (downloaded at runtime)
+*.onnx
+*.pth
+*.pt
+*.bin
+*.safetensors
+*.whl
+
+# Logs
+*.log
+logs/
+
+# IDE / OS
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+desktop.ini
@@ -0,0 +1,21 @@
+FROM python:3.10-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libgl1 libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY pyproject.toml README.md ./
+COPY src ./src
+
+RUN pip install --upgrade pip \
+    && pip install .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "openocr_markdown.api:app", "--host", "0.0.0.0", "--port", "8000"]
@@ -0,0 +1,97 @@
+# OpenOCR Markdown Service
+
+一个精简版图片/PDF 转 Markdown 服务，核心依赖 `openocr-python==0.1.5`，保留文档解析能力，去掉训练、评测和实验脚本。
+
+## 功能
+
+- `POST /api/recognize`：上传图片或 PDF，生成 Markdown 和 JSON。
+- `GET /api/history`：查看已解析文件列表。
+- `GET /output/...`：访问生成的 Markdown、JSON 和原始文件。
+- `openocr-md`：命令行直接解析单个图片/PDF。
+
+## 本地运行
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .
+cp .env.example .env
+uvicorn openocr_markdown.api:app --host 0.0.0.0 --port 8000
+```
+
+Windows PowerShell:
+
+```powershell
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1
+pip install -e .
+Copy-Item .env.example .env
+uvicorn openocr_markdown.api:app --host 0.0.0.0 --port 8000
+```
+
+首次运行会自动下载 OpenOCR 所需模型，耗时取决于网络和机器性能。
+
+## API 示例
+
+```bash
+curl -F "file=@examples/page.png" http://127.0.0.1:8000/api/recognize
+curl http://127.0.0.1:8000/api/history
+```
+
+重复上传同名文件默认复用已有结果；需要重新识别时增加 `force=true`：
+
+```bash
+curl -F "file=@examples/page.png" -F "force=true" http://127.0.0.1:8000/api/recognize
+```
+
+## 命令行
+
+```bash
+openocr-md path/to/document.pdf --output-dir data/output
+```
+
+## Docker 部署
+
+```bash
+cp .env.example .env
+docker compose up -d --build
+```
+
+服务启动后访问：
+
+- API 文档：`http://服务器IP:8000/docs`
+- 健康检查：`http://服务器IP:8000/health`
+- 输出文件：`http://服务器IP:8000/output/...`
+
+## 关键配置
+
+通过 `.env` 或服务器环境变量配置：
+
+```env
+OPENOCR_OUTPUT_DIR=data/output
+OPENOCR_UPLOAD_DIR=data/uploads
+OPENOCR_USE_GPU=false
+OPENOCR_AUTO_DOWNLOAD=true
+OPENOCR_LAYOUT_THRESHOLD=0.4
+OPENOCR_MAX_LENGTH=2048
+OPENOCR_CORS_ORIGINS=*
+```
+
+如果远程服务器有 GPU，可以先安装带 GPU 的运行环境，再把 `OPENOCR_USE_GPU=true`。当前 Dockerfile 默认是 CPU 版，适合先验证完整链路。
+
+## 项目结构
+
+```text
+.
+├── src/openocr_markdown/
+│   ├── api.py        # FastAPI 服务
+│   ├── cli.py        # 命令行入口
+│   ├── config.py     # 环境变量配置
+│   ├── engine.py     # OpenOCR 模型加载和解析封装
+│   └── schemas.py    # API 响应模型
+├── tests/
+├── Dockerfile
+├── docker-compose.yml
+├── pyproject.toml
+└── .env.example
+```
@@ -0,0 +1,11 @@
+services:
+  openocr-markdown:
+    build: .
+    container_name: openocr-markdown
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+    env_file:
+      - .env
+    volumes:
+      - ./data:/app/data
@@ -0,0 +1,41 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openocr-markdown-service"
+version = "0.1.0"
+description = "FastAPI service that converts images or PDFs to Markdown with OpenOCR."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.30",
+    "python-multipart>=0.0.9",
+    "pydantic-settings>=2.4",
+    "openocr-python==0.1.5",
+]
+
+[project.optional-dependencies]
+server = [
+    "torch",
+    "torchvision",
+]
+dev = [
+    "pytest>=8.0",
+    "ruff>=0.6",
+]
+
+[project.scripts]
+openocr-md-api = "openocr_markdown.api:main"
+openocr-md = "openocr_markdown.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
@@ -0,0 +1,3 @@
+"""OpenOCR Markdown service package."""
+
+__version__ = "0.1.0"
@@ -0,0 +1,156 @@
+import re
+import shutil
+import time
+from pathlib import Path
+from uuid import uuid4
+
+import uvicorn
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+
+from .config import settings
+from .engine import get_doc_parser, parse_to_outputs
+from .schemas import HistoryItem, HistoryResponse, RecognizeResponse
+
+
+SUPPORTED_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".pdf"}
+
+
+def safe_stem(filename: str) -> str:
+    stem = Path(filename).stem.strip()
+    stem = re.sub(r"[^a-zA-Z0-9._-]+", "_", stem)
+    return stem.strip("._-") or f"document_{uuid4().hex[:8]}"
+
+
+def find_first(path: Path, patterns: tuple[str, ...]) -> Path | None:
+    for pattern in patterns:
+        match = next(path.glob(pattern), None)
+        if match:
+            return match
+    return None
+
+
+def create_app() -> FastAPI:
+    settings.output_dir.mkdir(parents=True, exist_ok=True)
+    settings.upload_dir.mkdir(parents=True, exist_ok=True)
+
+    app = FastAPI(title=settings.app_name)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origin_list,
+        allow_credentials=settings.cors_origin_list != ["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @app.on_event("startup")
+    async def warm_up_model() -> None:
+        get_doc_parser()
+
+    @app.get("/", include_in_schema=False)
+    async def root() -> RedirectResponse:
+        return RedirectResponse(url="/docs")
+
+    @app.get("/health")
+    async def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    @app.get("/api/history", response_model=HistoryResponse)
+    async def history() -> HistoryResponse:
+        items: list[HistoryItem] = []
+        for item_dir in settings.output_dir.iterdir():
+            if not item_dir.is_dir():
+                continue
+            markdown_file = find_first(item_dir, ("*.md",))
+            if not markdown_file:
+                continue
+            json_file = find_first(item_dir, ("*.json",))
+            source_file = find_first(item_dir, ("origin_*",))
+            items.append(
+                HistoryItem(
+                    id=item_dir.name,
+                    markdown_url=f"/output/{item_dir.name}/{markdown_file.name}",
+                    json_url=f"/output/{item_dir.name}/{json_file.name}" if json_file else None,
+                    source_url=f"/output/{item_dir.name}/{source_file.name}" if source_file else None,
+                    updated_at=markdown_file.stat().st_mtime,
+                )
+            )
+        items.sort(key=lambda item: item.updated_at, reverse=True)
+        return HistoryResponse(data=items)
+
+    @app.post("/api/recognize", response_model=RecognizeResponse)
+    async def recognize(
+        file: UploadFile = File(...),
+        force: bool = Form(default=False),
+    ) -> RecognizeResponse:
+        if not file.filename:
+            raise HTTPException(status_code=400, detail="Missing filename.")
+
+        suffix = Path(file.filename).suffix.lower()
+        if suffix not in SUPPORTED_SUFFIXES:
+            raise HTTPException(status_code=400, detail=f"Unsupported file type: {suffix}")
+
+        doc_id = safe_stem(file.filename)
+        output_dir = settings.output_dir / doc_id
+        markdown_file = output_dir / f"{doc_id}.md"
+
+        if output_dir.exists():
+            if force:
+                shutil.rmtree(output_dir)
+            elif markdown_file.exists():
+                return RecognizeResponse(
+                    id=doc_id,
+                    status="success",
+                    message="Existing result reused. Set force=true to re-run.",
+                    markdown_url=f"/output/{doc_id}/{markdown_file.name}",
+                    json_url=f"/output/{doc_id}/{doc_id}.json",
+                )
+
+        output_dir.mkdir(parents=True, exist_ok=True)
+        upload_path = settings.upload_dir / f"{doc_id}{suffix}"
+        source_path = output_dir / f"origin_{Path(file.filename).name}"
+
+        with upload_path.open("wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+
+        started_at = time.time()
+        try:
+            parse_to_outputs(upload_path, settings.output_dir)
+            if upload_path.exists():
+                shutil.move(str(upload_path), source_path)
+        except Exception as exc:
+            if upload_path.exists():
+                upload_path.unlink()
+            raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+        elapsed = round(time.time() - started_at, 2)
+        return RecognizeResponse(
+            id=doc_id,
+            status="success",
+            message=f"Parsed successfully in {elapsed}s.",
+            markdown_url=f"/output/{doc_id}/{doc_id}.md",
+            json_url=f"/output/{doc_id}/{doc_id}.json",
+            source_url=f"/output/{doc_id}/{source_path.name}",
+        )
+
+    app.mount("/output", StaticFiles(directory=settings.output_dir), name="output")
+    return app
+
+
+app = create_app()
+
+
+def main() -> None:
+    uvicorn.run(
+        "openocr_markdown.api:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=False,
+        log_level=settings.log_level,
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,28 @@
+import argparse
+from pathlib import Path
+
+from .config import settings
+from .engine import parse_to_outputs
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Convert an image or PDF to Markdown with OpenOCR.")
+    parser.add_argument("input_path", type=Path, help="Image or PDF path.")
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=settings.output_dir,
+        help="Directory where OpenOCR outputs are written.",
+    )
+    args = parser.parse_args()
+
+    if not args.input_path.exists():
+        raise SystemExit(f"Input file does not exist: {args.input_path}")
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    parse_to_outputs(args.input_path, args.output_dir)
+    print(f"Done. Results written under: {args.output_dir}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,33 @@
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Runtime settings loaded from environment variables."""
+
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_name: str = "OpenOCR Markdown Service"
+    output_dir: Path = Field(default=Path("data/output"), alias="OPENOCR_OUTPUT_DIR")
+    upload_dir: Path = Field(default=Path("data/uploads"), alias="OPENOCR_UPLOAD_DIR")
+    use_gpu: bool = Field(default=False, alias="OPENOCR_USE_GPU")
+    auto_download: bool = Field(default=True, alias="OPENOCR_AUTO_DOWNLOAD")
+    layout_threshold: float = Field(default=0.4, alias="OPENOCR_LAYOUT_THRESHOLD")
+    max_length: int = Field(default=2048, alias="OPENOCR_MAX_LENGTH")
+    cors_origins: str = Field(default="*", alias="OPENOCR_CORS_ORIGINS")
+    log_level: Literal["debug", "info", "warning", "error"] = Field(
+        default="info",
+        alias="OPENOCR_LOG_LEVEL",
+    )
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        if self.cors_origins.strip() == "*":
+            return ["*"]
+        return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()]
+
+
+settings = Settings()
@@ -0,0 +1,31 @@
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from openocr import OpenOCR
+
+from .config import settings
+
+
+@lru_cache(maxsize=1)
+def get_doc_parser() -> OpenOCR:
+    """Load the document parser once per worker process."""
+
+    return OpenOCR(
+        task="doc",
+        use_gpu=settings.use_gpu,
+        auto_download=settings.auto_download,
+        use_layout_detection=True,
+    )
+
+
+def parse_to_outputs(input_path: Path, output_dir: Path) -> dict[str, Any]:
+    parser = get_doc_parser()
+    result = parser(
+        image_path=str(input_path),
+        layout_threshold=settings.layout_threshold,
+        max_length=settings.max_length,
+    )
+    parser.save_to_markdown(result, str(output_dir))
+    parser.save_to_json(result, str(output_dir))
+    return result
@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+
+class RecognizeResponse(BaseModel):
+    id: str
+    status: str
+    message: str
+    markdown_url: str | None = None
+    json_url: str | None = None
+    source_url: str | None = None
+
+
+class HistoryItem(BaseModel):
+    id: str
+    markdown_url: str
+    json_url: str | None = None
+    source_url: str | None = None
+    updated_at: float
+
+
+class HistoryResponse(BaseModel):
+    data: list[HistoryItem]
@@ -0,0 +1,9 @@
+from openocr_markdown.api import safe_stem
+
+
+def test_safe_stem_keeps_simple_names() -> None:
+    assert safe_stem("invoice-01.png") == "invoice-01"
+
+
+def test_safe_stem_replaces_unsafe_names() -> None:
+    assert safe_stem("../report 01!.pdf") == "report_01"