PythonとStreamlitを活用したドキュメント検索システムの構築方法 (ChatGPT API 不要)

pip install streamlit pymupdf python-docx openpyxl python-pptx scikit-learn

document_search_app/
│
├── app.py                   # メインアプリケーション
├── requirements.txt         # 必要なライブラリ一覧
└── utils/
    ├── file_loader.py       # ファイル読み込みとテキスト抽出
    └── text_search.py       # 検索アルゴリズム

import os
import fitz  # PyMuPDF for PDF
import docx
import openpyxl
from pptx import Presentation

def extract_text_from_pdf(file_path):
    text = ""
    with fitz.open(file_path) as pdf:
        for page in pdf:
            text += page.get_text()
    return text

def extract_text_from_word(file_path):
    doc = docx.Document(file_path)
    return "\n".join([para.text for para in doc.paragraphs])

def extract_text_from_excel(file_path):
    workbook = openpyxl.load_workbook(file_path)
    text = []
    for sheet in workbook.sheetnames:
        worksheet = workbook[sheet]
        for row in worksheet.iter_rows(values_only=True):
            text.append(" ".join(str(cell) for cell in row if cell))
    return "\n".join(text)

def extract_text_from_powerpoint(file_path):
    presentation = Presentation(file_path)
    text = []
    for slide in presentation.slides:
        for shape in slide.shapes:
            if hasattr(shape, "text"):
                text.append(shape.text)
    return "\n".join(text)

def extract_text(file_path):
    if file_path.lower().endswith('.pdf'):
        return extract_text_from_pdf(file_path)
    elif file_path.lower().endswith('.docx'):
        return extract_text_from_word(file_path)
    elif file_path.lower().endswith('.xlsx'):
        return extract_text_from_excel(file_path)
    elif file_path.lower().endswith('.pptx'):
        return extract_text_from_powerpoint(file_path)
    else:
        return None

def load_documents_from_directory(directory):
    documents = {}
    for root, _, files in os.walk(directory):
        for file in files:
            if file.startswith('~$'):  # 一時ファイルを無視
                continue
            file_path = os.path.join(root, file)
            if file.lower().endswith(('.pdf', '.docx', '.xlsx', '.pptx')):
                try:
                    text = extract_text(file_path)
                    if text:
                        documents[file_path] = text
                except Exception as e:
                    print(f"Error reading {file_path}: {e}")
    return documents

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

def search_documents(query, documents):
    doc_texts = list(documents.values())
    file_paths = list(documents.keys())
    
    vectorizer = TfidfVectorizer()
    tfidf_matrix = vectorizer.fit_transform(doc_texts + [query])
    
    query_vector = tfidf_matrix[-1]
    doc_vectors = tfidf_matrix[:-1]
    similarities = cosine_similarity(query_vector, doc_vectors).flatten()
    
    sorted_indices = np.argsort(-similarities)
    results = [(file_paths[i], similarities[i]) for i in sorted_indices if similarities[i] > 0]
    
    return results

import streamlit as st
from utils.file_loader import load_documents_from_directory
from utils.text_search import search_documents

st.title("ドキュメント検索システム")

# ディレクトリパスの入力
directory = st.text_input("検索対象のディレクトリを入力してください")

if directory:
    if st.button("読み込み"):
        documents = load_documents_from_directory(directory)
        if documents:
            st.success(f"{len(documents)} 件のドキュメントを読み込みました。")
            
            query = st.text_input("検索クエリを入力してください")
            
            if st.button("検索"):
                if query:
                    results = search_documents(query, documents)
                    if results:
                        st.subheader("検索結果")
                        for file_path, score in results:
                            st.write(f"ファイル: {file_path} - 類似度: {score:.2f}")
                    else:
                        st.info("該当する結果が見つかりませんでした。")
        else:
            st.error("ドキュメントが見つかりませんでした。")

streamlit run app.py