How to Build an AI Meeting Assistant That Takes Notes and Tracks Action Items

ai-meeting-assistant/
├── app/
│   ├── __init__.py
│   ├── aligner.py
│   ├── transcriber.py
│   └── extractor.py
├── tests/
│   └── test_meeting_assistant.py
└── requirements.txt

pip install faster-whisper==1.2.1 pydantic==2.9.2 openai==1.51.0 pytest==8.3.3

from typing import List
from pydantic import BaseModel, Field

class WordSegment(BaseModel):
    start: float
    end: float
    word: str

class SpeakerTurn(BaseModel):
    start: float
    end: float
    speaker: str

class Utterance(BaseModel):
    speaker: str
    start: float
    end: float
    text: str

def align_words_with_speakers(
    words: List[WordSegment],
    speaker_turns: List[SpeakerTurn],
) -> List[Utterance]:
    """Assigns each transcribed word to the speaker turn with maximum temporal overlap and merges consecutive turns."""
    if not words:
        return []

    assigned_words = []
    for w in words:
        best_speaker = "SPEAKER_UNKNOWN"
        max_overlap = 0.0
        w_mid = (w.start + w.end) / 2.0

        for turn in speaker_turns:
            overlap = max(0.0, min(w.end, turn.end) - max(w.start, turn.start))
            if overlap > max_overlap:
                max_overlap = overlap
                best_speaker = turn.speaker
            elif max_overlap == 0.0 and turn.start <= w_mid <= turn.end:
                best_speaker = turn.speaker

        assigned_words.append((best_speaker, w))

    # Merge contiguous words spoken by the same speaker into utterances
    utterances: List[Utterance] = []
    curr_speaker, first_w = assigned_words[0]
    curr_start = first_w.start
    curr_end = first_w.end
    curr_tokens = [first_w.word.strip()]

    for speaker, w in assigned_words[1:]:
        if speaker == curr_speaker:
            curr_end = w.end
            curr_tokens.append(w.word.strip())
        else:
            utterances.append(
                Utterance(
                    speaker=curr_speaker,
                    start=round(curr_start, 2),
                    end=round(curr_end, 2),
                    text=" ".join(curr_tokens),
                )
            )
            curr_speaker = speaker
            curr_start = w.start
            curr_end = w.end
            curr_tokens = [w.word.strip()]

    utterances.append(
        Utterance(
            speaker=curr_speaker,
            start=round(curr_start, 2),
            end=round(curr_end, 2),
            text=" ".join(curr_tokens),
        )
    )
    return utterances

from typing import List
from faster_whisper import WhisperModel
from app.aligner import WordSegment

class LocalWhisperTranscriber:
    def __init__(self, model_size: str = "base", device: str = "cpu", compute_type: str = "int8"):
        self.model = WhisperModel(model_size, device=device, compute_type=compute_type)

    def transcribe_words(self, audio_path: str) -> List[WordSegment]:
        """Transcribes audio with Silero VAD filtering and word-level timestamps."""
        segments, _info = self.model.transcribe(
            audio_path,
            beam_size=5,
            word_timestamps=True,
            vad_filter=True,
            vad_parameters=dict(min_silence_duration_ms=500),
        )
        words: List[WordSegment] = []
        for seg in segments:
            if seg.words:
                for w in seg.words:
                    words.append(WordSegment(start=w.start, end=w.end, word=w.word))
        return words

from typing import List, Optional
from pydantic import BaseModel, Field
from openai import AsyncOpenAI
from app.aligner import Utterance

class ActionItem(BaseModel):
    owner: str = Field(..., description="Speaker name or ID responsible for the task.")
    task_description: str = Field(..., description="Concrete action item to complete.")
    due_date_iso: Optional[str] = Field(None, description="ISO-8601 date if mentioned, else None.")
    evidence_timestamp: str = Field(..., description="Start-end timestamp in [MM:SS] where agreed.")

class MeetingNotes(BaseModel):
    executive_summary: str
    key_decisions: List[str]
    action_items: List[ActionItem]
    open_questions: List[str]

def format_transcript_for_llm(utterances: List[Utterance]) -> str:
    lines = []
    for u in utterances:
        lines.append(f"[{u.start:.1f}s - {u.end:.1f}s] {u.speaker}: {u.text}")
    return "\n".join(lines)

async def extract_meeting_notes(
    client: AsyncOpenAI,
    utterances: List[Utterance],
    meeting_date_iso: str,
    model: str = "gpt-4o-mini",
) -> MeetingNotes:
    formatted_text = format_transcript_for_llm(utterances)
    prompt = (
        f"Meeting Reference Date: {meeting_date_iso}\n"
        "Extract an executive summary, key decisions, and speaker-attributed action items from this transcript.\n"
        "Resolve relative dates (like 'next Friday') against the Meeting Reference Date.\n\n"
        f"Transcript:\n{formatted_text}"
    )
    completion = await client.beta.chat.completions.parse(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        response_format=MeetingNotes,
        temperature=0.0,
    )
    return completion.choices[0].message.parsed

from app.aligner import WordSegment, SpeakerTurn, align_words_with_speakers

def test_temporal_word_to_speaker_alignment():
    words = [
        WordSegment(start=0.0, end=0.4, word="I"),
        WordSegment(start=0.4, end=0.8, word="will"),
        WordSegment(start=0.8, end=1.5, word="deploy"),
        WordSegment(start=2.0, end=2.5, word="Sounds"),
        WordSegment(start=2.5, end=3.0, word="good."),
    ]
    turns = [
        SpeakerTurn(start=0.0, end=1.6, speaker="Aniket"),
        SpeakerTurn(start=1.9, end=3.2, speaker="Priya"),
    ]
    utterances = align_words_with_speakers(words, turns)
    assert len(utterances) == 2
    assert utterances[0].speaker == "Aniket"
    assert utterances[0].text == "I will deploy"
    assert utterances[1].speaker == "Priya"
    assert utterances[1].text == "Sounds good."