path: root/tools/perf/python/export-to-sqlite.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
"""
Export perf data to a sqlite3 database.

This script has been ported to use the modern perf Python module and the
standard library sqlite3 module. It no longer requires PySide2 or QtSql
for exporting.

Examples of using this script with Intel PT:

	$ perf record -e intel_pt//u ls
	$ python export-to-sqlite.py -i perf.data -o pt_example

To browse the database, sqlite3 can be used e.g.

	$ sqlite3 pt_example
	sqlite> .header on
	sqlite> select * from samples_view where id < 10;
	sqlite> .mode column
	sqlite> select * from samples_view where id < 10;
	sqlite> .tables
	sqlite> .schema samples_view
	sqlite> .quit

An example of using the database is provided by the script
exported-sql-viewer.py. Refer to that script for details.

Ported from tools/perf/scripts/python/export-to-sqlite.py
"""

from __future__ import annotations
import typing
import argparse
import os
import shutil
import sqlite3
import struct
import sys
import tempfile
from typing import Dict, Optional
import perf


PERF_IP_FLAG_BRANCH = 1 << 0
PERF_IP_FLAG_CALL = 1 << 1
PERF_IP_FLAG_RETURN = 1 << 2
PERF_IP_FLAG_CONDITIONAL = 1 << 3
PERF_IP_FLAG_SYSCALLRET = 1 << 4
PERF_IP_FLAG_ASYNC = 1 << 5
PERF_IP_FLAG_INTERRUPT = 1 << 6
PERF_IP_FLAG_TX_ABORT = 1 << 7
PERF_IP_FLAG_TRACE_BEGIN = 1 << 8
PERF_IP_FLAG_TRACE_END = 1 << 9
PERF_IP_FLAG_IN_TX = 1 << 10
PERF_IP_FLAG_VMENTRY = 1 << 11
PERF_IP_FLAG_VMEXIT = 1 << 12

BRANCH_TYPES = [
    (0, "no branch"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"),
    (PERF_IP_FLAG_BRANCH, "unconditional jump"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, "software interrupt"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, "return from interrupt"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, "system call"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET,
     "return from system call"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_INTERRUPT,
     "hardware interrupt"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"),
    (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"),
]
for _b_type, _b_name in list(BRANCH_TYPES):
    if (_b_type == PERF_IP_FLAG_BRANCH or
            (_b_type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))):
        continue
    BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_BEGIN, f"trace begin / {_b_name}"))
    BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_END, f"{_b_name} / trace end"))


class DatabaseExporter:
    """Handles database connection and exporting of perf events."""

    def __init__(self, db_path: str):
        self.con = sqlite3.connect(db_path)
        self.con.execute("PRAGMA journal_mode = MEMORY")
        self.session: Optional[perf.session] = None
        self.sample_count = 0

        # Caches and counters grouped to reduce instance attributes
        self.caches: Dict[str, dict] = {
            'machines': {},
            'threads': {},
            'comms': {},
            'dsos': {},
            'symbols': {},
            'events': {},
            'branch_types': {},
            'call_paths': {}
        }

        self.next_id = {
            'machine': 1,
            'thread': 1,
            'comm': 1,
            'dso': 1,
            'symbol': 1,
            'event': 1,
            'branch_type': 1,
            'call_path': 1
        }

        self.create_tables()

    def create_tables(self) -> None:
        """Create database tables."""
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS selected_events (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    name    VARCHAR(80))
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS machines (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    pid     INTEGER,
                    root_dir VARCHAR(4096))
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS threads (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    machine_id BIGINT,
                    process_id BIGINT,
                    pid     INTEGER,
                    tid     INTEGER)
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS comms (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    comm    VARCHAR(16),
                    c_thread_id BIGINT,
                    c_time  BIGINT,
                    exec_flag BOOLEAN)
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS comm_threads (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    comm_id BIGINT,
                    thread_id BIGINT)
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS dsos (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    machine_id BIGINT,
                    short_name VARCHAR(256),
                    long_name VARCHAR(4096),
                    build_id VARCHAR(64))
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS symbols (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    dso_id  BIGINT,
                    sym_start BIGINT,
                    sym_end BIGINT,
                    binding INTEGER,
                    name    VARCHAR(2048))
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS branch_types (
                    id      INTEGER         NOT NULL        PRIMARY KEY,
                    name    VARCHAR(80))
        """)
        self.con.execute("""
            CREATE TABLE IF NOT EXISTS samples (
                    id              INTEGER         NOT NULL        PRIMARY KEY,
                    evsel_id        BIGINT,
                    machine_id      BIGINT,
                    thread_id       BIGINT,
                    comm_id         BIGINT,
                    dso_id          BIGINT,
                    symbol_id       BIGINT,
                    sym_offset      BIGINT,
                    ip              BIGINT,
                    time            BIGINT,
                    cpu             INTEGER,
                    to_dso_id       BIGINT,
                    to_symbol_id    BIGINT,
                    to_sym_offset   BIGINT,
                    to_ip           BIGINT,
                    period          BIGINT,
                    weight          BIGINT,
                    transaction_    BIGINT,
                    data_src        BIGINT,
                    branch_type     INTEGER,
                    in_tx           BOOLEAN,
                    call_path_id    BIGINT,
                    insn_count      BIGINT,
                    cyc_count       BIGINT,
                    flags           INTEGER)
        """)
        self.con.execute('''
            CREATE TABLE IF NOT EXISTS calls (
                id          INTEGER     NOT NULL    PRIMARY KEY,
                thread_id   BIGINT,
                comm_id     BIGINT,
                call_path_id BIGINT,
                call_time   BIGINT,
                return_time BIGINT,
                branch_count BIGINT,
                call_id     BIGINT,
                return_id   BIGINT,
                parent_call_path_id BIGINT,
                flags       INTEGER,
                parent_id   BIGINT,
                insn_count  BIGINT,
                cyc_count   BIGINT
            )
        ''')
        self.con.execute('''
            CREATE TABLE IF NOT EXISTS call_paths (
                id          INTEGER     NOT NULL    PRIMARY KEY,
                parent_id   BIGINT,
                symbol_id   BIGINT,
                ip          BIGINT
            )
        ''')

        self.con.execute('''
            CREATE TABLE IF NOT EXISTS cbr (
                id integer primary key,
                cbr integer,
                mhz integer,
                percent integer
            )
        ''')
        self.con.execute('''
            CREATE TABLE IF NOT EXISTS mwait (
                id integer primary key,
                hints integer,
                extensions integer
            )
        ''')

        self.con.execute('''
            CREATE TABLE IF NOT EXISTS context_switches (
                id integer primary key,
                machine_id bigint,
                time bigint,
                cpu integer,
                thread_out_id bigint,
                comm_out_id bigint,
                thread_in_id bigint,
                comm_in_id bigint,
                flags integer
            )
        ''')
        self.con.execute(
            "CREATE TABLE IF NOT EXISTS ptwrite ("
            "id integer primary key, payload integer, exact_ip integer"
            ")"
        )
        self.con.execute(
            "CREATE TABLE IF NOT EXISTS pwre ("
            "id integer primary key, hw integer, cstate integer, subcstate integer, "
            "hw_name text, cstate_name text"
            ")"
        )
        self.con.execute(
            "CREATE TABLE IF NOT EXISTS exstop ("
            "id integer primary key, exact_ip integer"
            ")"
        )
        self.con.execute(
            "CREATE TABLE IF NOT EXISTS pwrx ("
            "id integer primary key, deepest_cstate integer, last_cstate integer, "
            "wake_reason integer"
            ")"
        )

        self.con.execute(
            "CREATE VIEW IF NOT EXISTS machines_view AS "
            "SELECT id, pid, root_dir, "
            "CASE WHEN id=0 THEN 'unknown' WHEN pid=-1 THEN 'host' ELSE 'guest' END "
            "AS host_or_guest FROM machines"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS dsos_view AS "
            "SELECT id, machine_id, "
            "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, "
            "short_name, long_name, build_id FROM dsos"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS symbols_view AS "
            "SELECT id, name, (SELECT short_name FROM dsos WHERE id=dso_id) AS dso, "
            "dso_id, sym_start, sym_end, "
            "CASE WHEN binding=0 THEN 'local' WHEN binding=1 THEN 'global' ELSE 'weak' END "
            "AS binding FROM symbols"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS threads_view AS "
            "SELECT id, machine_id, "
            "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, "
            "process_id, pid, tid FROM threads"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS comm_threads_view AS "
            "SELECT comm_id, (SELECT comm FROM comms WHERE id = comm_id) AS command, "
            "thread_id, (SELECT pid FROM threads WHERE id = thread_id) AS pid, "
            "(SELECT tid FROM threads WHERE id = thread_id) AS tid FROM comm_threads"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS call_paths_view AS "
            "SELECT c.id, printf('%x', c.ip) AS ip, c.symbol_id, "
            "(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol, "
            "(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id, "
            "(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name, "
            "c.parent_id, printf('%x', p.ip) AS parent_ip, p.symbol_id AS parent_symbol_id, "
            "(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol, "
            "(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id, "
            "(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name "
            "FROM call_paths c LEFT JOIN call_paths p ON p.id = c.parent_id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS calls_view AS "
            "SELECT calls.id, thread_id, "
            "(SELECT pid FROM threads WHERE id = thread_id) AS pid, "
            "(SELECT tid FROM threads WHERE id = thread_id) AS tid, "
            "(SELECT comm FROM comms WHERE id = comm_id) AS command, "
            "call_path_id, printf('%x', ip) AS ip, symbol_id, "
            "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, "
            "call_time, return_time, return_time - call_time AS elapsed_time, "
            "branch_count, insn_count, cyc_count, "
            "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) "
            "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, "
            "call_id, return_id, "
            "CASE WHEN flags=0 THEN '' WHEN flags=1 THEN 'no call' "
            "WHEN flags=2 THEN 'no return' WHEN flags=3 THEN 'no call/return' "
            "WHEN flags IN (4, 6) THEN 'jmp' ELSE flags END AS flags, "
            "parent_call_path_id, calls.parent_id "
            "FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS samples_view AS "
            "SELECT id, time, cpu, "
            "(SELECT pid FROM threads WHERE id = thread_id) AS pid, "
            "(SELECT tid FROM threads WHERE id = thread_id) AS tid, "
            "(SELECT comm FROM comms WHERE id = comm_id) AS command, "
            "(SELECT name FROM selected_events WHERE id = evsel_id) AS event, "
            "printf('%x', ip) AS ip_hex, "
            "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, sym_offset, "
            "(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name, "
            "printf('%x', to_ip) AS to_ip_hex, "
            "(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol, to_sym_offset, "
            "(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name, "
            "(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name, "
            "in_tx, call_path_id, insn_count, cyc_count, "
            "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) "
            "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, flags FROM samples"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS ptwrite_view AS "
            "SELECT ptwrite.id, time, cpu, printf('%x', payload) AS payload_hex, "
            "CASE WHEN exact_ip=0 THEN 'False' ELSE 'True' END AS exact_ip "
            "FROM ptwrite INNER JOIN samples ON samples.id = ptwrite.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS cbr_view AS "
            "SELECT cbr.id, time, cpu, cbr, mhz, percent "
            "FROM cbr INNER JOIN samples ON samples.id = cbr.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS mwait_view AS "
            "SELECT mwait.id, time, cpu, printf('%x', hints) AS hints_hex, "
            "printf('%x', extensions) AS extensions_hex "
            "FROM mwait INNER JOIN samples ON samples.id = mwait.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS pwre_view AS "
            "SELECT pwre.id, time, cpu, cstate, subcstate, "
            "CASE WHEN hw=0 THEN 'False' ELSE 'True' END AS hw "
            "FROM pwre INNER JOIN samples ON samples.id = pwre.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS exstop_view AS "
            "SELECT exstop.id, time, cpu, "
            "CASE WHEN exact_ip=0 THEN 'False' ELSE 'True' END AS exact_ip "
            "FROM exstop INNER JOIN samples ON samples.id = exstop.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS pwrx_view AS "
            "SELECT pwrx.id, time, cpu, deepest_cstate, last_cstate, "
            "CASE WHEN wake_reason=1 THEN 'Interrupt' "
            "WHEN wake_reason=2 THEN 'Timer Deadline' "
            "WHEN wake_reason=4 THEN 'Monitored Address' "
            "WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' "
            "ELSE wake_reason END AS wake_reason "
            "FROM pwrx INNER JOIN samples ON samples.id = pwrx.id"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS power_events_view AS "
            "SELECT samples.id, time, cpu, selected_events.name AS event, "
            "CASE WHEN selected_events.name='cbr' THEN "
            "(SELECT cbr FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS cbr, "
            "CASE WHEN selected_events.name='cbr' THEN "
            "(SELECT mhz FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS mhz, "
            "CASE WHEN selected_events.name='cbr' THEN "
            "(SELECT percent FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS percent, "
            "CASE WHEN selected_events.name='mwait' THEN "
            "(SELECT printf('%x', hints) FROM mwait WHERE mwait.id = samples.id) "
            "ELSE \"\" END AS hints_hex, "
            "CASE WHEN selected_events.name='mwait' THEN "
            "(SELECT printf('%x', extensions) FROM mwait WHERE mwait.id = samples.id) "
            "ELSE \"\" END AS extensions_hex, "
            "CASE WHEN selected_events.name='pwre' THEN "
            "(SELECT cstate FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS cstate, "
            "CASE WHEN selected_events.name='pwre' THEN "
            "(SELECT subcstate FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS subcstate, "
            "CASE WHEN selected_events.name='pwre' THEN "
            "(SELECT hw FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS hw, "
            "CASE WHEN selected_events.name='exstop' THEN "
            "(SELECT exact_ip FROM exstop WHERE exstop.id = samples.id) "
            "ELSE \"\" END AS exact_ip, "
            "CASE WHEN selected_events.name='pwrx' THEN "
            "(SELECT deepest_cstate FROM pwrx WHERE pwrx.id = samples.id) "
            "ELSE \"\" END AS deepest_cstate, "
            "CASE WHEN selected_events.name='pwrx' THEN "
            "(SELECT last_cstate FROM pwrx WHERE pwrx.id = samples.id) "
            "ELSE \"\" END AS last_cstate, "
            "CASE WHEN selected_events.name='pwrx' THEN "
            "(SELECT wake_reason FROM pwrx WHERE pwrx.id = samples.id) "
            "ELSE \"\" END AS wake_reason "
            "FROM samples INNER JOIN selected_events ON selected_events.id = evsel_id "
            "WHERE selected_events.name IN ('cbr','mwait','pwre','exstop','pwrx')"
        )
        self.con.execute(
            "CREATE VIEW IF NOT EXISTS context_switches_view AS "
            "SELECT context_switches.id, context_switches.machine_id, "
            "context_switches.time, context_switches.cpu, "
            "th_out.pid AS pid_out, th_out.tid AS tid_out, comm_out.comm AS comm_out, "
            "th_in.pid AS pid_in, th_in.tid AS tid_in, comm_in.comm AS comm_in, "
            "CASE WHEN flags=0 THEN 'in' WHEN flags=1 THEN 'out' "
            "WHEN flags=3 THEN 'out preempt' ELSE flags END AS flags "
            "FROM context_switches "
            "INNER JOIN threads AS th_out ON th_out.id = context_switches.thread_out_id "
            "INNER JOIN threads AS th_in  ON th_in.id  = context_switches.thread_in_id "
            "INNER JOIN comms AS comm_out ON comm_out.id = context_switches.comm_out_id "
            "INNER JOIN comms AS comm_in  ON comm_in.id  = context_switches.comm_in_id"
        )

        # id == 0 means unknown. It is easier to create records for them than
        # replace the zeroes with NULLs
        self.con.execute("INSERT OR IGNORE INTO selected_events VALUES (0, 'unknown')")
        self.con.execute("INSERT OR IGNORE INTO machines VALUES (0, 0, 'unknown')")
        self.con.execute("INSERT OR IGNORE INTO threads VALUES (0, 0, 0, -1, -1)")
        self.con.execute("INSERT OR IGNORE INTO comms VALUES (0, 'unknown', 0, 0, 0)")
        self.con.execute("INSERT OR IGNORE INTO dsos VALUES (0, 0, 'unknown', 'unknown', '')")
        self.con.execute("INSERT OR IGNORE INTO call_paths VALUES (0, 0, 0, 0)")
        self.con.execute("INSERT OR IGNORE INTO symbols VALUES (0, 0, 0, 0, 0, 'unknown')")
        for b_type, b_name in BRANCH_TYPES:
            self.con.execute("INSERT OR IGNORE INTO branch_types VALUES (?, ?)", (b_type, b_name))
            self.caches['branch_types'][b_type] = b_name

        self.caches['events']['unknown'] = 0
        self.caches['threads'][(0, -1, -1)] = 0
        self.caches['threads'][(0, 0, 0)] = 0
        self.caches['comms'][('unknown', 0)] = 0
        self.caches['dsos'][(0, 'unknown', 'unknown', '')] = 0
        self.caches['symbols'][(0, 'unknown')] = (0, 0, 0)
        # Initialize comm_threads mapping
        self.comm_threads_cache: set[tuple[int, int]] = set()
        self.next_comm_thread_id = 1

    def _exec(self, sql: str, params: tuple[typing.Any, ...] = ()) -> sqlite3.Cursor:
        """Execute SQL statement converting unsigned 64-bit ints to signed 64-bit.

        Python's sqlite3 module only binds signed 64-bit integers (int64_t) and
        raises OverflowError for unsigned 64-bit kernel addresses (>= 2^63).
        Convert values in [2^63, 2^64) into two's-complement signed 64-bit ints.
        """
        conv_params = tuple(
            p - 0x10000000000000000 if isinstance(p, int) and p >= 0x8000000000000000 else p
            for p in params
        )
        return self.con.execute(sql, conv_params)

    def get_machine_id(self, machine_pid: Optional[int]) -> int:
        """Get or create machine ID."""
        if machine_pid is None or machine_pid <= 0 or machine_pid > 0x7fffffff:
            machine_pid = -1
        if machine_pid in self.caches['machines']:
            return self.caches['machines'][machine_pid]
        machine_id = self.next_id['machine']
        self.next_id['machine'] += 1
        self._exec("INSERT INTO machines VALUES (?, ?, ?)",
                   (machine_id, machine_pid, ""))
        self.caches['machines'][machine_pid] = machine_id
        return machine_id

    def get_event_id(self, name: str) -> int:
        """Get or create event ID."""
        if name in self.caches['events']:
            return self.caches['events'][name]
        event_id = self.next_id['event']
        self._exec("INSERT INTO selected_events VALUES (?, ?)",
                   (event_id, name))
        self.caches['events'][name] = event_id
        self.next_id['event'] += 1
        return event_id

    def get_thread_id(self, machine_id: int, pid: Optional[int], tid: Optional[int]) -> int:
        """Get or create thread ID."""
        if pid is None:
            pid = -1
        elif pid > 0x7fffffff:
            pid -= 0x100000000
        if tid is None:
            tid = -1
        elif tid > 0x7fffffff:
            tid -= 0x100000000
        key = (machine_id, pid, tid)
        if key in self.caches['threads']:
            return self.caches['threads'][key]
        thread_id = self.next_id['thread']
        self.next_id['thread'] += 1
        self.caches['threads'][key] = thread_id
        process_id = thread_id if pid == tid else self.get_thread_id(machine_id, pid, pid)
        self._exec("INSERT INTO threads VALUES (?, ?, ?, ?, ?)",
                   (thread_id, machine_id, process_id, pid, tid))
        return thread_id

    def get_comm_id(self, comm: str, thread_id: int) -> int:
        """Get or create comm ID."""
        key = (comm, thread_id)
        if key not in self.caches['comms']:
            comm_id = self.next_id['comm']
            self._exec("INSERT INTO comms VALUES (?, ?, ?, ?, ?)",
                       (comm_id, comm, thread_id, 0, 0))
            self.caches['comms'][key] = comm_id
            self.next_id['comm'] += 1
        comm_id = self.caches['comms'][key]
        mapping_key = (comm_id, thread_id)
        if mapping_key not in self.comm_threads_cache:
            comm_thread_id = self.next_comm_thread_id
            self._exec("INSERT INTO comm_threads VALUES (?, ?, ?)",
                       (comm_thread_id, comm_id, thread_id))
            self.comm_threads_cache.add(mapping_key)
            self.next_comm_thread_id += 1
        return comm_id

    def get_dso_id(self, short_name: str, long_name: str,
                   build_id: str, machine_id: int = 0) -> int:
        """Get or create DSO ID."""
        key = (machine_id, short_name, long_name, build_id)
        if key in self.caches['dsos']:
            return self.caches['dsos'][key]
        short_key = (machine_id, short_name)
        if not build_id and short_key in self.caches['dsos']:
            return self.caches['dsos'][short_key]
        dso_id = self.next_id['dso']
        self._exec("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)",
                   (dso_id, machine_id, short_name, long_name, build_id))
        self.caches['dsos'][key] = dso_id
        self.caches['dsos'][short_key] = dso_id
        self.next_id['dso'] += 1
        return dso_id

    def get_symbol_id(self, dso_id: int, name: str, start: int,
                      end: int) -> int:
        """Get or create symbol ID."""
        key = (dso_id, name)
        if key in self.caches['symbols']:
            symbol_id, cached_start, cached_end = self.caches['symbols'][key]
            if cached_start == 0 and cached_end == 0 and (start != 0 or end != 0):
                self._exec("UPDATE symbols SET sym_start = ?, sym_end = ? WHERE id = ?",
                           (start, end, symbol_id))
                self.caches['symbols'][key] = (symbol_id, start, end)
            return symbol_id
        symbol_id = self.next_id['symbol']
        self._exec("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)",
                   (symbol_id, dso_id, start, end, 0, name))
        self.caches['symbols'][key] = (symbol_id, start, end)
        self.next_id['symbol'] += 1
        return symbol_id

    def get_call_path_id(self, parent_id: int, symbol_id: int,
                         ip: int) -> int:
        """Get or create call path ID."""
        key = (parent_id, symbol_id, ip)
        if key in self.caches['call_paths']:
            return self.caches['call_paths'][key]
        call_path_id = self.next_id['call_path']
        self._exec("INSERT INTO call_paths VALUES (?, ?, ?, ?)",
                   (call_path_id, parent_id, symbol_id, ip))
        self.caches['call_paths'][key] = call_path_id
        self.next_id['call_path'] += 1
        return call_path_id

    def process_event(self, sample: perf.sample_event) -> None:
        """Callback for processing events."""

        machine_pid = getattr(sample, 'machine_pid', 0)
        machine_id = self.get_machine_id(machine_pid)
        thread_id = self.get_thread_id(machine_id, sample.sample_pid, sample.sample_tid)

        comm = "Unknown_comm"
        try:
            if self.session is not None:
                proc = self.session.find_thread(sample.sample_pid, sample.sample_tid)
                if proc:
                    comm_name = proc.comm()
                    if comm_name:
                        comm = comm_name
        except TypeError:
            pass
        comm_id = self.get_comm_id(comm, thread_id)

        build_id = sample.dso_bid or ""
        dso_id = self.get_dso_id(
            sample.dso or "Unknown_dso",
            sample.dso_long_name or "Unknown_dso_long",
            build_id,
            machine_id
        )

        symbol_id = self.get_symbol_id(
            dso_id,
            sample.symbol or "Unknown_symbol",
            sample.sym_start or 0,
            sample.sym_end or 0
        )

        # Handle callchain
        call_path_id = 0
        if hasattr(sample, 'callchain') and sample.callchain:
            parent_id = 0
            for node in reversed(sample.callchain):
                dso_name = node.dso or "Unknown_dso"
                symbol_name = node.symbol or "Unknown_symbol"

                node_dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id)
                node_symbol_id = self.get_symbol_id(node_dso_id, symbol_name, 0, 0)

                parent_id = self.get_call_path_id(parent_id, node_symbol_id, node.ip)
            call_path_id = parent_id
        else:
            call_path_id = 0

        # Insert sample
        event_name_str = str(sample.evsel)
        if event_name_str.startswith("evsel(") and event_name_str.endswith(")"):
            event_name_str = event_name_str[6:-1]
        cursor = self._exec("""
            INSERT INTO samples VALUES (
                NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
            )
        """, (
            self.get_event_id(event_name_str),
            machine_id, thread_id, comm_id,
            dso_id, symbol_id,
            getattr(sample, 'sym_offset', 0) or 0,
            sample.sample_ip, sample.sample_time, sample.sample_cpu,
            self.get_dso_id(sample.addr_dso or "Unknown_dso",
                            sample.addr_dso or "Unknown_dso_long", "", machine_id),
            self.get_symbol_id(
                self.get_dso_id(sample.addr_dso or "Unknown_dso",
                                sample.addr_dso or "Unknown_dso_long", "", machine_id),
                sample.addr_symbol or "Unknown_symbol", 0, 0
            ),
            sample.addr_sym_offset or 0,
            sample.sample_addr or 0,
            sample.sample_period or 0,
            sample.sample_weight or 0,
            sample.transaction or 0,
            sample.sample_data_src,
            sample.branch_type or 0,
            sample.in_tx or 0,
            call_path_id,
            sample.sample_insn_count,
            sample.sample_cyc_count,
            getattr(sample, "flags", 0)  # flags
        ))
        sample_id = cursor.lastrowid

        # Handle Intel PT specific raw payloads mathematically equivalent to
        # legacy synth_data unpacking

        if event_name_str == "ptwrite" and hasattr(sample, "raw_buf"):
            try:
                flags, payload = struct.unpack_from("<IQ", sample.raw_buf)
                self._exec("INSERT INTO ptwrite VALUES (?, ?, ?)",
                           (sample_id, payload, flags & 1))
            except struct.error:
                pass

        elif event_name_str == "cbr" and hasattr(sample, "raw_buf"):
            try:
                data = struct.unpack_from("<BBBBII", sample.raw_buf)
                cbr_val, max_nonturbo, freq = data[0], data[2], data[4]
                MHz = (freq + 500) // 1000
                percent = ((cbr_val * 1000 // max_nonturbo) + 5) // 10 if max_nonturbo else 0
                self._exec("INSERT INTO cbr VALUES (?, ?, ?, ?)",
                           (sample_id, cbr_val, MHz, percent))
            except struct.error:
                pass

        elif event_name_str == "mwait" and hasattr(sample, "raw_buf"):
            try:
                flags, payload = struct.unpack_from("<IQ", sample.raw_buf)
                hints = payload & 0xff
                extensions = (payload >> 32) & 0x3
                self._exec("INSERT INTO mwait VALUES (?, ?, ?)",
                           (sample_id, hints, extensions))
            except struct.error:
                pass

        elif event_name_str == "pwre" and hasattr(sample, "raw_buf"):
            try:
                flags, payload = struct.unpack_from("<IQ", sample.raw_buf)
                hw = (payload >> 7) & 1
                cstate = (payload >> 12) & 0xf
                subcstate = (payload >> 8) & 0xf
                self._exec("INSERT INTO pwre (id, cstate, subcstate, hw) VALUES (?, ?, ?, ?)",
                           (sample_id, cstate, subcstate, hw))
            except struct.error:
                pass

        elif event_name_str == "exstop" and hasattr(sample, "raw_buf"):
            try:
                flags = struct.unpack_from("<I", sample.raw_buf)[0]
                self._exec("INSERT INTO exstop VALUES (?, ?)",
                           (sample_id, flags & 1))
            except struct.error:
                pass

        elif event_name_str == "pwrx" and hasattr(sample, "raw_buf"):
            try:
                flags, payload = struct.unpack_from("<IQ", sample.raw_buf)
                deepest_cstate = payload & 0xf
                last_cstate = (payload >> 4) & 0xf
                wake_reason = (payload >> 8) & 0xf
                self._exec("INSERT INTO pwrx VALUES (?, ?, ?, ?)",
                           (sample_id, deepest_cstate, last_cstate, wake_reason))
            except struct.error:
                pass

        self.sample_count += 1
        if self.sample_count % 10000 == 0:
            self.commit()

    def get_call_path_ids(self, call_path: typing.Any, machine_id: int = 0) -> tuple[int, int]:
        """Add a perf.callchain as call_paths rows, return its id and parent's."""
        parent_id = 0
        call_path_id = 0
        for node in call_path:
            dso_name = node.dso or "Unknown_dso"
            symbol_name = node.symbol or "Unknown_symbol"
            dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id)
            symbol_id = self.get_symbol_id(dso_id, symbol_name, 0, 0)
            parent_id = call_path_id
            call_path_id = self.get_call_path_id(parent_id, symbol_id, node.ip)
        return call_path_id, parent_id

    def process_call_return(self, cr: perf.call_return) -> None:
        """Callback for processing call_return events."""
        machine_id = self.get_machine_id(getattr(cr, "machine_pid", None))
        thread_id = self.get_thread_id(machine_id, cr.pid, cr.tid)
        comm_id = self.get_comm_id(cr.comm or "Unknown_comm", thread_id)
        call_path_id, parent_call_path_id = self.get_call_path_ids(cr.call_path or [], machine_id)
        # calls.id and calls.parent_id use the db_id of the perf module, as a
        # call is given an id before it returns its parent cannot be numbered
        # here.
        self._exec(
            "INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (cr.db_id, thread_id, comm_id, call_path_id,
             cr.call_time, cr.return_time,
             cr.branch_count, cr.call_ref,
             cr.return_ref, parent_call_path_id,
             cr.flags, cr.parent_id,
             cr.insn_count, cr.cyc_count)
        )

    def process_context_switch(self, event: typing.Any) -> None:
        """Callback for processing context switch events."""
        misc = getattr(event, 'misc', 0)
        out = bool(misc & (1 << 13)) # PERF_RECORD_MISC_SWITCH_OUT
        out_preempt = bool(misc & (1 << 14)) # PERF_RECORD_MISC_SWITCH_OUT_PREEMPT
        flags = (1 if out else 0) | ((1 if out_preempt else 0) << 1)
        machine_pid = getattr(event, 'machine_pid', None)
        machine_id = self.get_machine_id(machine_pid)

        sample_pid = getattr(event, 'sample_pid', None)
        if sample_pid is None:
            sample_pid = -1
        elif sample_pid > 0x7fffffff:
            sample_pid -= 0x100000000
        sample_tid = getattr(event, 'sample_tid', None)
        if sample_tid is None:
            sample_tid = -1
        elif sample_tid > 0x7fffffff:
            sample_tid -= 0x100000000
        next_prev_pid = getattr(event, 'next_prev_pid', None)
        if next_prev_pid is None:
            next_prev_pid = -1
        elif next_prev_pid > 0x7fffffff:
            next_prev_pid -= 0x100000000
        next_prev_tid = getattr(event, 'next_prev_tid', None)
        if next_prev_tid is None:
            next_prev_tid = -1
        elif next_prev_tid > 0x7fffffff:
            next_prev_tid -= 0x100000000

        th_a_id = self.get_thread_id(machine_id, sample_pid, sample_tid)
        comm_a = "Unknown_comm"
        if self.session:
            try:
                proc = self.session.find_thread(sample_pid, sample_tid)
                if proc:
                    comm_a_name = proc.comm()
                    if comm_a_name:
                        comm_a = comm_a_name
            except TypeError:
                pass
        comm_a_id = self.get_comm_id(comm_a, th_a_id)

        th_b_id = self.get_thread_id(machine_id, next_prev_pid, next_prev_tid)
        comm_b = "Unknown_comm"
        if self.session:
            try:
                proc = self.session.find_thread(next_prev_pid, next_prev_tid)
                if proc:
                    comm_b_name = proc.comm()
                    if comm_b_name:
                        comm_b = comm_b_name
            except TypeError:
                pass
        comm_b_id = self.get_comm_id(comm_b, th_b_id)

        if out:
            th_out_id = th_a_id
            comm_out_id = comm_a_id
            th_in_id = th_b_id
            comm_in_id = comm_b_id
        else:
            th_out_id = th_b_id
            comm_out_id = comm_b_id
            th_in_id = th_a_id
            comm_in_id = comm_a_id

        self._exec(
            "INSERT INTO context_switches VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)",
            (machine_id, getattr(event, 'sample_time', 0), getattr(event, 'sample_cpu', 0) or 0,
             th_out_id, comm_out_id, th_in_id, comm_in_id, flags)
        )

    def finish(self) -> None:
        """Create indexes and update summary metadata after processing events.

        pcpid_idx, pid_idx, and comms.has_calls are required by
        exported-sql-viewer.py to filter the command dropdown to threads with
        call graphs and to expand parent/child call trees efficiently.
        """
        self.con.execute("CREATE INDEX IF NOT EXISTS pcpid_idx ON calls (parent_call_path_id)")
        self.con.execute("CREATE INDEX IF NOT EXISTS pid_idx ON calls (parent_id)")
        self.con.execute("ALTER TABLE comms ADD has_calls boolean")
        self.con.execute("UPDATE comms SET has_calls = 1 WHERE comms.id IN "
                         "(SELECT DISTINCT comm_id FROM calls)")

    def commit(self) -> None:
        """Commit transaction."""
        self.con.commit()

    def close(self) -> None:
        """Close connection."""
        self.con.close()


if __name__ == "__main__":
    ap = argparse.ArgumentParser(
        description="Export perf data to a sqlite3 database")
    ap.add_argument("-i", "--input", default="perf.data",
                    help="Input file name")
    ap.add_argument("-o", "--output", default="perf.db",
                    help="Output database name")
    ap.add_argument("--itrace", help="Instruction Tracing options (e.g. crpt)")
    args = ap.parse_args()

    try:
        fd = os.open(args.output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
    except FileExistsError:
        print(f"Error: {args.output} already exists")
        sys.exit(1)

    tmp_dir = None
    exporter = None
    succeeded = False
    try:
        tmp_dir = tempfile.mkdtemp(prefix="perf-export-sqlite-")
        tmp_db = os.path.join(tmp_dir, "perf.db")
        exporter = DatabaseExporter(tmp_db)
        session = perf.session(perf.data(args.input),
                               sample=exporter.process_event,
                               context_switch=exporter.process_context_switch,
                               call_return=exporter.process_call_return,
                               itrace=args.itrace)
        exporter.session = session
        try:
            session.process_events()
        finally:
            # Break the C perf.session -> bound method -> exporter reference cycle.
            exporter.session = None
        exporter.finish()
        exporter.commit()
        exporter.close()
        exporter = None
        with open(tmp_db, "rb") as src, os.fdopen(fd, "wb", closefd=False) as dst:
            shutil.copyfileobj(src, dst)
        succeeded = True
        print(f"Successfully exported to {args.output}")
    except (OSError, RuntimeError, ValueError, sqlite3.Error):
        import traceback
        traceback.print_exc()
    finally:
        if exporter is not None:
            exporter.session = None
            try:
                exporter.close()
            except (OSError, sqlite3.Error):
                pass
        try:
            os.close(fd)
        except OSError:
            pass
        if tmp_dir is not None:
            shutil.rmtree(tmp_dir, ignore_errors=True)
        if not succeeded:
            try:
                os.remove(args.output)
            except OSError:
                pass
    if not succeeded:
        sys.exit(1)