# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""SQLite coverage data."""

from __future__ import annotations

import collections
import datetime
import functools
import glob
import itertools
import os
import random
import socket
import sqlite3
import string
import sys
import textwrap
import threading
import uuid
import zlib
from collections.abc import Collection, Mapping, Sequence
from typing import Any, Callable, cast

from coverage.debug import NoDebugging, auto_repr, file_summary
from coverage.exceptions import CoverageException, DataError
from coverage.misc import file_be_gone, isolate_module
from coverage.numbits import numbits_to_nums, numbits_union, nums_to_numbits
from coverage.sqlitedb import SqliteDb
from coverage.types import AnyCallable, FilePath, TArc, TDebugCtl, TLineNo, TWarnFn
from coverage.version import __version__

os = isolate_module(os)

# If you change the schema: increment the SCHEMA_VERSION and update the
# docs in docs/dbschema.rst by running "make cogdoc".

SCHEMA_VERSION = 7

# Schema versions:
# 1: Released in 5.0a2
# 2: Added contexts in 5.0a3.
# 3: Replaced line table with line_map table.
# 4: Changed line_map.bitmap to line_map.numbits.
# 5: Added foreign key declarations.
# 6: Key-value in meta.
# 7: line_map -> line_bits

SCHEMA = """\
CREATE TABLE coverage_schema (
    -- One row, to record the version of the schema in this db.
    version integer
);

CREATE TABLE meta (
    -- Key-value pairs, to record metadata about the data
    key text,
    value text,
    unique (key)
    -- Possible keys:
    --  'has_arcs' boolean      -- Is this data recording branches?
    --  'sys_argv' text         -- The coverage command line that recorded the data.
    --  'version' text          -- The version of coverage.py that made the file.
    --  'when' text             -- Datetime when the file was created.
);

CREATE TABLE file (
    -- A row per file measured.
    id integer primary key,
    path text,
    unique (path)
);

CREATE TABLE context (
    -- A row per context measured.
    id integer primary key,
    context text,
    unique (context)
);

CREATE TABLE line_bits (
    -- If recording lines, a row per context per file executed.
    -- All of the line numbers for that file/context are in one numbits.
    file_id integer,            -- foreign key to `file`.
    context_id integer,         -- foreign key to `context`.
    numbits blob,               -- see the numbits functions in coverage.numbits
    foreign key (file_id) references file (id),
    foreign key (context_id) references context (id),
    unique (file_id, context_id)
);

CREATE TABLE arc (
    -- If recording branches, a row per context per from/to line transition executed.
    file_id integer,            -- foreign key to `file`.
    context_id integer,         -- foreign key to `context`.
    fromno integer,             -- line number jumped from.
    tono integer,               -- line number jumped to.
    foreign key (file_id) references file (id),
    foreign key (context_id) references context (id),
    unique (file_id, context_id, fromno, tono)
);

CREATE TABLE tracer (
    -- A row per file indicating the tracer used for that file.
    file_id integer primary key,
    tracer text,
    foreign key (file_id) references file (id)
);
"""


def _locked(method: AnyCallable) -> AnyCallable:
    """A decorator for methods that should hold self._lock."""

    @functools.wraps(method)
    def _wrapped(self: CoverageData, *args: Any, **kwargs: Any) -> Any:
        if self._debug.should("lock"):
            self._debug.write(f"Locking {self._lock!r} for {method.__name__}")
        with self._lock:
            if self._debug.should("lock"):
                self._debug.write(f"Locked {self._lock!r} for {method.__name__}")
            return method(self, *args, **kwargs)

    return _wrapped


class NumbitsUnionAgg:
    """SQLite aggregate function for computing union of numbits."""

    def __init__(self) -> None:
        self.result = b""

    def step(self, value: bytes) -> None:
        """Process one value in the aggregation."""
        self.result = numbits_union(self.result, value)

    def finalize(self) -> bytes:
        """Return the final aggregated result."""
        return self.result


class CoverageData:
    """Manages collected coverage data, including file storage.

    This class is the public supported API to the data that coverage.py
    collects during program execution.  It includes information about what code
    was executed. It does not include information from the analysis phase, to
    determine what lines could have been executed, or what lines were not
    executed.

    .. note::

        The data file is currently a SQLite database file, with a
        :ref:`documented schema <dbschema>`. The schema is subject to change
        though, so be careful about querying it directly. Use this API if you
        can to isolate yourself from changes.

    There are a number of kinds of data that can be collected:

    * **lines**: the line numbers of source lines that were executed.
      These are always available.

    * **arcs**: pairs of source and destination line numbers for transitions
      between source lines.  These are only available if branch coverage was
      used.

    * **file tracer names**: the module names of the file tracer plugins that
      handled each file in the data.

    Lines, arcs, and file tracer names are stored for each source file. File
    names in this API are case-sensitive, even on platforms with
    case-insensitive file systems.

    A data file either stores lines, or arcs, but not both.

    A data file is associated with the data when the :class:`CoverageData`
    is created, using the parameters `basename`, `suffix`, and `no_disk`. The
    base name can be queried with :meth:`base_filename`, and the actual file
    name being used is available from :meth:`data_filename`.

    To read an existing coverage.py data file, use :meth:`read`.  You can then
    access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`,
    or :meth:`file_tracer`.

    The :meth:`has_arcs` method indicates whether arc data is available.  You
    can get a set of the files in the data with :meth:`measured_files`.  As
    with most Python containers, you can determine if there is any data at all
    by using this object as a boolean value.

    The contexts for each line in a file can be read with
    :meth:`contexts_by_lineno`.

    To limit querying to certain contexts, use :meth:`set_query_context` or
    :meth:`set_query_contexts`. These will narrow the focus of subsequent
    :meth:`lines`, :meth:`arcs`, and :meth:`contexts_by_lineno` calls. The set
    of all measured context names can be retrieved with
    :meth:`measured_contexts`.

    Most data files will be created by coverage.py itself, but you can use
    methods here to create data files if you like.  The :meth:`add_lines`,
    :meth:`add_arcs`, and :meth:`add_file_tracers` methods add data, in ways
    that are convenient for coverage.py.

    To record data for contexts, use :meth:`set_context` to set a context to
    be used for subsequent :meth:`add_lines` and :meth:`add_arcs` calls.

    To add a source file without any measured data, use :meth:`touch_file`,
    or :meth:`touch_files` for a list of such files.

    Write the data to its file with :meth:`write`.

    You can clear the data in memory with :meth:`erase`.  Data for specific
    files can be removed from the database with :meth:`purge_files`.

    Two data collections can be combined by using :meth:`update` on one
    :class:`CoverageData`, passing it the other.

    Data in a :class:`CoverageData` can be serialized and deserialized with
    :meth:`dumps` and :meth:`loads`.

    The methods used during the coverage.py collection phase
    (:meth:`add_lines`, :meth:`add_arcs`, :meth:`set_context`, and
    :meth:`add_file_tracers`) are thread-safe.  Other methods may not be.

    """

    def __init__(
        self,
        basename: FilePath | None = None,
        suffix: str | bool | None = None,
        no_disk: bool = False,
        warn: TWarnFn | None = None,
        debug: TDebugCtl | None = None,
    ) -> None:
        """Create a :class:`CoverageData` object to hold coverage-measured data.

        Arguments:
            basename (str): the base name of the data file, defaulting to
                ".coverage". This can be a path to a file in another directory.
            suffix (str or bool): has the same meaning as the `data_suffix`
                argument to :class:`coverage.Coverage`.
            no_disk (bool): if True, keep all data in memory, and don't
                write any disk file.
            warn: a warning callback function, accepting a warning message
                argument.
            debug: a `DebugControl` object (optional)

        """
        self._no_disk = no_disk
        self._basename = os.path.abspath(basename or ".coverage")
        self._suffix = suffix
        self._warn = warn
        self._debug = debug or NoDebugging()

        self._choose_filename()
        # Maps filenames to row ids.
        self._file_map: dict[str, int] = {}
        # Maps thread ids to SqliteDb objects.
        self._dbs: dict[int, SqliteDb] = {}
        self._pid = os.getpid()
        # Synchronize the operations used during collection.
        self._lock = threading.RLock()

        # Are we in sync with the data file?
        self._have_used = False

        self._has_lines = False
        self._has_arcs = False

        self._current_context: str | None = None
        self._current_context_id: int | None = None
        self._query_context_ids: list[int] | None = None

    __repr__ = auto_repr

    def _debug_dataio(self, msg: str, filename: str) -> None:
        """A helper for debug messages which are all similar."""
        if self._debug.should("dataio"):
            self._debug.write(f"{msg} {filename!r} ({file_summary(filename)})")

    def _choose_filename(self) -> None:
        """Set self._filename based on inited attributes."""
        if self._no_disk:
            self._filename = f"file:coverage-{uuid.uuid4()}?mode=memory&cache=shared"
        else:
            self._filename = self._basename
            suffix = filename_suffix(self._suffix)
            if suffix:
                self._filename += f".{suffix}"

    def _reset(self) -> None:
        """Reset our attributes."""
        if not self._no_disk:
            self.close()
        self._file_map = {}
        self._have_used = False
        self._current_context_id = None

    def close(self, force: bool = False) -> None:
        """Really close all the database objects."""
        if self._debug.should("dataio"):
            self._debug.write(f"Closing dbs, force={force}: {self._dbs}")
        for db in self._dbs.values():
            db.close(force=force)
        self._dbs = {}

    def _open_db(self) -> None:
        """Open an existing db file, and read its metadata."""
        self._debug_dataio("Opening data file", self._filename)
        self._dbs[threading.get_ident()] = SqliteDb(self._filename, self._debug, self._no_disk)
        self._read_db()

    def _read_db(self) -> None:
        """Read the metadata from a database so that we are ready to use it."""
        with self._dbs[threading.get_ident()] as db:
            try:
                row = db.execute_one("select version from coverage_schema")
                assert row is not None
            except Exception as exc:
                if "no such table: coverage_schema" in str(exc):
                    self._init_db(db)
                else:
                    raise DataError(
                        "Data file {!r} doesn't seem to be a coverage data file: {}".format(
                            self._filename,
                            exc,
                        ),
                    ) from exc
            else:
                schema_version = row[0]
                if schema_version != SCHEMA_VERSION:
                    raise DataError(
                        "Couldn't use data file {!r}: wrong schema: {} instead of {}".format(
                            self._filename,
                            schema_version,
                            SCHEMA_VERSION,
                        ),
                    )

            row = db.execute_one("select value from meta where key = 'has_arcs'")
            if row is not None:
                self._has_arcs = bool(int(row[0]))
                self._has_lines = not self._has_arcs

            with db.execute("select id, path from file") as cur:
                for file_id, path in cur:
                    self._file_map[path] = file_id

    def _init_db(self, db: SqliteDb) -> None:
        """Write the initial contents of the database."""
        self._debug_dataio("Initing data file", self._filename)
        db.executescript(SCHEMA)
        db.execute_void("INSERT INTO coverage_schema (version) VALUES (?)", (SCHEMA_VERSION,))

        # When writing metadata, avoid information that will needlessly change
        # the hash of the data file, unless we're debugging processes.
        meta_data = [
            ("version", __version__),
        ]
        if self._debug.should("process"):
            meta_data.extend(
                [
                    ("sys_argv", str(getattr(sys, "argv", None))),
                    ("when", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
                ]
            )
        db.executemany_void("INSERT OR IGNORE INTO meta (key, value) VALUES (?, ?)", meta_data)

    def _connect(self) -> SqliteDb:
        """Get the SqliteDb object to use."""
        if threading.get_ident() not in self._dbs:
            self._open_db()
        return self._dbs[threading.get_ident()]

    def __bool__(self) -> bool:
        if threading.get_ident() not in self._dbs and not os.path.exists(self._filename):
            return False
        try:
            with self._connect() as con:
                with con.execute("SELECT * FROM file LIMIT 1") as cur:
                    return bool(list(cur))
        except CoverageException:
            return False

    def dumps(self) -> bytes:
        """Serialize the current data to a byte string.

        The format of the serialized data is not documented. It is only
        suitable for use with :meth:`loads` in the same version of
        coverage.py.

        Note that this serialization is not what gets stored in coverage data
        files.  This method is meant to produce bytes that can be transmitted
        elsewhere and then deserialized with :meth:`loads`.

        Returns:
            A byte string of serialized data.

        .. versionadded:: 5.0

        """
        self._debug_dataio("Dumping data from data file", self._filename)
        with self._connect() as con:
            script = con.dump()
            return b"z" + zlib.compress(script.encode("utf-8"))

    def loads(self, data: bytes) -> None:
        """Deserialize data from :meth:`dumps`.

        Use with a newly-created empty :class:`CoverageData` object.  It's
        undefined what happens if the object already has data in it.

        Note that this is not for reading data from a coverage data file.  It
        is only for use on data you produced with :meth:`dumps`.

        Arguments:
            data: A byte string of serialized data produced by :meth:`dumps`.

        .. versionadded:: 5.0

        """
        self._debug_dataio("Loading data into data file", self._filename)
        if data[:1] != b"z":
            raise DataError(
                f"Unrecognized serialization: {data[:40]!r} (head of {len(data)} bytes)",
            )
        script = zlib.decompress(data[1:]).decode("utf-8")
        self._dbs[threading.get_ident()] = db = SqliteDb(self._filename, self._debug, self._no_disk)
        with db:
            db.executescript(script)
        self._read_db()
        self._have_used = True

    def _file_id(self, filename: str, add: bool = False) -> int | None:
        """Get the file id for `filename`.

        If filename is not in the database yet, add it if `add` is True.
        If `add` is not True, return None.
        """
        if filename not in self._file_map:
            if add:
                with self._connect() as con:
                    self._file_map[filename] = con.execute_for_rowid(
                        "INSERT OR REPLACE INTO file (path) VALUES (?)",
                        (filename,),
                    )
        return self._file_map.get(filename)

    def _context_id(self, context: str) -> int | None:
        """Get the id for a context."""
        assert context is not None
        self._start_using()
        with self._connect() as con:
            row = con.execute_one("SELECT id FROM context WHERE context = ?", (context,))
            if row is not None:
                return cast(int, row[0])
            else:
                return None

    @_locked
    def set_context(self, context: str | None) -> None:
        """Set the current context for future :meth:`add_lines` etc.

        `context` is a str, the name of the context to use for the next data
        additions.  The context persists until the next :meth:`set_context`.

        .. versionadded:: 5.0

        """
        if self._debug.should("dataop"):
            self._debug.write(f"Setting coverage context: {context!r}")
        self._current_context = context
        self._current_context_id = None

    def _set_context_id(self) -> None:
        """Use the _current_context to set _current_context_id."""
        context = self._current_context or ""
        context_id = self._context_id(context)
        if context_id is not None:
            self._current_context_id = context_id
        else:
            with self._connect() as con:
                self._current_context_id = con.execute_for_rowid(
                    "INSERT INTO context (context) VALUES (?)",
                    (context,),
                )

    def base_filename(self) -> str:
        """The base filename for storing data.

        .. versionadded:: 5.0

        """
        return self._basename

    def data_filename(self) -> str:
        """Where is the data stored?

        .. versionadded:: 5.0

        """
        return self._filename

    @_locked
    def add_lines(self, line_data: Mapping[str, Collection[TLineNo]]) -> None:
        """Add measured line data.

        `line_data` is a dictionary mapping file names to iterables of ints::

            { filename: { line1, line2, ... }, ...}

        """
        if self._debug.should("dataop"):
            self._debug.write(
                "Adding lines: %d files, %d lines total"
                % (
                    len(line_data),
                    sum(len(lines) for lines in line_data.values()),
                )
            )
            if self._debug.should("dataop2"):
                for filename, linenos in sorted(line_data.items()):
                    self._debug.write(f"  {filename}: {linenos}")
        self._start_using()
        self._choose_lines_or_arcs(lines=True)
        if not line_data:
            return
        with self._connect() as con:
            self._set_context_id()
            for filename, linenos in line_data.items():
                line_bits = nums_to_numbits(linenos)
                file_id = self._file_id(filename, add=True)
                query = "SELECT numbits FROM line_bits WHERE file_id = ? AND context_id = ?"
                with con.execute(query, (file_id, self._current_context_id)) as cur:
                    existing = list(cur)
                if existing:
                    line_bits = numbits_union(line_bits, existing[0][0])

                con.execute_void(
                    """
                    INSERT OR REPLACE INTO line_bits
                    (file_id, context_id, numbits) VALUES (?, ?, ?)
                    """,
                    (file_id, self._current_context_id, line_bits),
                )

    @_locked
    def add_arcs(self, arc_data: Mapping[str, Collection[TArc]]) -> None:
        """Add measured arc data.

        `arc_data` is a dictionary mapping file names to iterables of pairs of
        ints::

            { filename: { (l1,l2), (l1,l2), ... }, ...}

        """
        if self._debug.should("dataop"):
            self._debug.write(
                "Adding arcs: %d files, %d arcs total"
                % (
                    len(arc_data),
                    sum(len(arcs) for arcs in arc_data.values()),
                )
            )
            if self._debug.should("dataop2"):
                for filename, arcs in sorted(arc_data.items()):
                    self._debug.write(f"  {filename}: {arcs}")
        self._start_using()
        self._choose_lines_or_arcs(arcs=True)
        if not arc_data:
            return
        with self._connect() as con:
            self._set_context_id()
            for filename, arcs in arc_data.items():
                if not arcs:
                    continue
                file_id = self._file_id(filename, add=True)
                data = [(file_id, self._current_context_id, fromno, tono) for fromno, tono in arcs]
                con.executemany_void(
                    """
                    INSERT OR IGNORE INTO arc
                    (file_id, context_id, fromno, tono) VALUES (?, ?, ?, ?)
                    """,
                    data,
                )

    def _choose_lines_or_arcs(self, lines: bool = False, arcs: bool = False) -> None:
        """Force the data file to choose between lines and arcs."""
        assert lines or arcs
        assert not (lines and arcs)
        if lines and self._has_arcs:
            if self._debug.should("dataop"):
                self._debug.write("Error: Can't add line measurements to existing branch data")
            raise DataError("Can't add line measurements to existing branch data")
        if arcs and self._has_lines:
            if self._debug.should("dataop"):
                self._debug.write("Error: Can't add branch measurements to existing line data")
            raise DataError("Can't add branch measurements to existing line data")
        if not self._has_arcs and not self._has_lines:
            self._has_lines = lines
            self._has_arcs = arcs
            with self._connect() as con:
                con.execute_void(
                    "INSERT OR IGNORE INTO meta (key, value) VALUES (?, ?)",
                    ("has_arcs", str(int(arcs))),
                )

    @_locked
    def add_file_tracers(self, file_tracers: Mapping[str, str]) -> None:
        """Add per-file plugin information.

        `file_tracers` is { filename: plugin_name, ... }

        """
        if self._debug.should("dataop"):
            self._debug.write(f"Adding file tracers: {len(file_tracers)} files")
        if not file_tracers:
            return
        self._start_using()
        with self._connect() as con:
            for filename, plugin_name in file_tracers.items():
                file_id = self._file_id(filename, add=True)
                existing_plugin = self.file_tracer(filename)
                if existing_plugin:
                    if existing_plugin != plugin_name:
                        raise DataError(
                            "Conflicting file tracer name for '{}': {!r} vs {!r}".format(
                                filename,
                                existing_plugin,
                                plugin_name,
                            ),
                        )
                elif plugin_name:
                    con.execute_void(
                        "INSERT INTO TRACER (file_id, tracer) VALUES (?, ?)",
                        (file_id, plugin_name),
                    )

    def touch_file(self, filename: str, plugin_name: str = "") -> None:
        """Ensure that `filename` appears in the data, empty if needed.

        `plugin_name` is the name of the plugin responsible for this file.
        It is used to associate the right filereporter, etc.
        """
        self.touch_files([filename], plugin_name)

    def touch_files(self, filenames: Collection[str], plugin_name: str | None = None) -> None:
        """Ensure that `filenames` appear in the data, empty if needed.

        `plugin_name` is the name of the plugin responsible for these files.
        It is used to associate the right filereporter, etc.
        """
        if self._debug.should("dataop"):
            self._debug.write(f"Touching {filenames!r}")
        self._start_using()
        with self._connect():  # Use this to get one transaction.
            if not self._has_arcs and not self._has_lines:
                raise DataError("Can't touch files in an empty CoverageData")

            for filename in filenames:
                self._file_id(filename, add=True)
                if plugin_name:
                    # Set the tracer for this file
                    self.add_file_tracers({filename: plugin_name})

    def purge_files(self, filenames: Collection[str]) -> None:
        """Purge any existing coverage data for the given `filenames`.

        .. versionadded:: 7.2

        """
        if self._debug.should("dataop"):
            self._debug.write(f"Purging data for {filenames!r}")
        self._start_using()
        with self._connect() as con:
            if self._has_lines:
                sql = "DELETE FROM line_bits WHERE file_id=?"
            elif self._has_arcs:
                sql = "DELETE FROM arc WHERE file_id=?"
            else:
                raise DataError("Can't purge files in an empty CoverageData")

            for filename in filenames:
                file_id = self._file_id(filename, add=False)
                if file_id is None:
                    continue
                con.execute_void(sql, (file_id,))

    def update(
        self,
        other_data: CoverageData,
        map_path: Callable[[str], str] | None = None,
    ) -> None:
        """Update this data with data from another :class:`CoverageData`.

        If `map_path` is provided, it's a function that re-map paths to match
        the local machine's.  Note: `map_path` is None only when called
        directly from the test suite.

        """
        if self._debug.should("dataop"):
            self._debug.write(
                "Updating with data from {!r}".format(
                    getattr(other_data, "_filename", "???"),
                )
            )
        if self._has_lines and other_data._has_arcs:
            raise DataError(
                "Can't combine branch coverage data with statement data", slug="cant-combine"
            )
        if self._has_arcs and other_data._has_lines:
            raise DataError(
                "Can't combine statement coverage data with branch data", slug="cant-combine"
            )

        map_path = map_path or (lambda p: p)

        # Force the database we're writing to to exist before we start nesting contexts.
        self._start_using()
        other_data.read()

        # Ensure other_data has a properly initialized database
        with other_data._connect():
            pass

        with self._connect() as con:
            assert con.con is not None
            con.con.isolation_level = "IMMEDIATE"

            # Register functions for SQLite
            con.con.create_function("numbits_union", 2, numbits_union)
            con.con.create_function("map_path", 1, map_path)
            con.con.create_aggregate(
                "numbits_union_agg",
                1,
                NumbitsUnionAgg,  # type: ignore[arg-type]
            )

            # Attach the other database
            con.execute_void("ATTACH DATABASE ? AS other_db", (other_data.data_filename(),))

            # Create temporary table with mapped file paths to avoid repeated map_path() calls
            con.execute_void("""
                CREATE TEMP TABLE other_file_mapped AS
                SELECT
                    other_file.id as other_file_id,
                    map_path(other_file.path) as mapped_path
                FROM other_db.file AS other_file
            """)

            # Check for tracer conflicts before proceeding
            with con.execute("""
                SELECT other_file_mapped.mapped_path,
                       COALESCE(main.tracer.tracer, ''),
                       COALESCE(other_db.tracer.tracer, '')
                FROM main.file
                LEFT JOIN main.tracer ON main.file.id = main.tracer.file_id
                INNER JOIN other_file_mapped ON main.file.path = other_file_mapped.mapped_path
                LEFT JOIN other_db.tracer ON other_file_mapped.other_file_id = other_db.tracer.file_id
                WHERE COALESCE(main.tracer.tracer, '') != COALESCE(other_db.tracer.tracer, '')
            """) as cur:
                conflicts = list(cur)
                if conflicts:
                    path, this_tracer, other_tracer = conflicts[0]
                    raise DataError(
                        "Conflicting file tracer name for '{}': {!r} vs {!r}".format(
                            path,
                            this_tracer,
                            other_tracer,
                        ),
                    )

            # Insert missing files from other_db (with map_path applied)
            con.execute_void("""
                INSERT OR IGNORE INTO main.file (path)
                SELECT DISTINCT mapped_path FROM other_file_mapped
            """)

            # Insert missing contexts from other_db
            con.execute_void("""
                INSERT OR IGNORE INTO main.context (context)
                SELECT context FROM other_db.context
            """)

            # Update file_map with any new files
            with con.execute("SELECT id, path FROM file") as cur:
                self._file_map.update({path: id for id, path in cur})

            with con.execute("""
                SELECT
                    EXISTS(SELECT 1 FROM other_db.arc),
                    EXISTS(SELECT 1 FROM other_db.line_bits)
            """) as cur:
                has_arcs, has_lines = cur.fetchone()

            # Handle arcs if present in other_db
            if has_arcs:
                self._choose_lines_or_arcs(arcs=True)

                # Create context mapping table for faster lookups
                con.execute_void("""
                    CREATE TEMP TABLE context_mapping AS
                    SELECT
                        other_context.id as other_id,
                        main_context.id as main_id
                    FROM other_db.context AS other_context
                    INNER JOIN main.context AS main_context ON other_context.context = main_context.context
                """)

                con.execute_void("""
                    INSERT OR IGNORE INTO main.arc (file_id, context_id, fromno, tono)
                    SELECT
                        main_file.id,
                        context_mapping.main_id,
                        other_arc.fromno,
                        other_arc.tono
                    FROM other_db.arc AS other_arc
                    INNER JOIN other_file_mapped ON other_arc.file_id = other_file_mapped.other_file_id
                    INNER JOIN context_mapping ON other_arc.context_id = context_mapping.other_id
                    INNER JOIN main.file AS main_file ON other_file_mapped.mapped_path = main_file.path
                """)

            # Handle line_bits if present in other_db
            if has_lines:
                self._choose_lines_or_arcs(lines=True)

                # Handle line_bits by aggregating other_db data by mapped target,
                # then inserting/updating
                con.execute_void("""
                    INSERT OR REPLACE INTO main.line_bits (file_id, context_id, numbits)
                    SELECT
                        main_file.id,
                        main_context.id,
                        numbits_union(
                            COALESCE((
                                SELECT numbits FROM main.line_bits
                                WHERE file_id = main_file.id AND context_id = main_context.id
                            ), X''),
                            aggregated.combined_numbits
                        )
                    FROM (
                        SELECT
                            other_file_mapped.mapped_path,
                            other_context.context,
                            numbits_union_agg(other_line_bits.numbits) as combined_numbits
                        FROM other_db.line_bits AS other_line_bits
                        INNER JOIN other_file_mapped ON other_line_bits.file_id = other_file_mapped.other_file_id
                        INNER JOIN other_db.context AS other_context ON other_line_bits.context_id = other_context.id
                        GROUP BY other_file_mapped.mapped_path, other_context.context
                    ) AS aggregated
                    INNER JOIN main.file AS main_file ON aggregated.mapped_path = main_file.path
                    INNER JOIN main.context AS main_context ON aggregated.context = main_context.context
                """)

            # Insert tracers from other_db (avoiding conflicts we already checked)
            con.execute_void("""
                INSERT OR IGNORE INTO main.tracer (file_id, tracer)
                SELECT
                    main_file.id,
                    other_tracer.tracer
                FROM other_db.tracer AS other_tracer
                INNER JOIN other_file_mapped ON other_tracer.file_id = other_file_mapped.other_file_id
                INNER JOIN main.file AS main_file ON other_file_mapped.mapped_path = main_file.path
            """)

        if not self._no_disk:
            # Update all internal cache data.
            self._reset()
            self.read()

    def erase(self, parallel: bool = False) -> None:
        """Erase the data in this object.

        If `parallel` is true, then also deletes data files created from the
        basename by parallel-mode.

        """
        self._reset()
        if self._no_disk:
            return
        self._debug_dataio("Erasing data file", self._filename)
        file_be_gone(self._filename)
        if parallel:
            data_dir, local = os.path.split(self._filename)
            local_abs_path = os.path.join(os.path.abspath(data_dir), local)
            pattern = glob.escape(local_abs_path) + ".*"
            for filename in glob.glob(pattern):
                self._debug_dataio("Erasing parallel data file", filename)
                file_be_gone(filename)

    def read(self) -> None:
        """Start using an existing data file."""
        if os.path.exists(self._filename):
            with self._connect():
                self._have_used = True

    def write(self) -> None:
        """Ensure the data is written to the data file."""
        self._debug_dataio("Writing (no-op) data file", self._filename)

    def _start_using(self) -> None:
        """Call this before using the database at all."""
        if self._pid != os.getpid():
            # Looks like we forked! Have to start a new data file.
            self._reset()
            self._choose_filename()
            self._pid = os.getpid()
        if not self._have_used:
            self.erase()
        self._have_used = True

    def has_arcs(self) -> bool:
        """Does the database have arcs (True) or lines (False)."""
        return bool(self._has_arcs)

    def measured_files(self) -> set[str]:
        """A set of all files that have been measured.

        Note that a file may be mentioned as measured even though no lines or
        arcs for that file are present in the data.

        """
        return set(self._file_map)

    def measured_contexts(self) -> set[str]:
        """A set of all contexts that have been measured.

        .. versionadded:: 5.0

        """
        self._start_using()
        with self._connect() as con:
            with con.execute("SELECT DISTINCT(context) FROM context") as cur:
                contexts = {row[0] for row in cur}
        return contexts

    def file_tracer(self, filename: str) -> str | None:
        """Get the plugin name of the file tracer for a file.

        Returns the name of the plugin that handles this file.  If the file was
        measured, but didn't use a plugin, then "" is returned.  If the file
        was not measured, then None is returned.

        """
        self._start_using()
        with self._connect() as con:
            file_id = self._file_id(filename)
            if file_id is None:
                return None
            row = con.execute_one("SELECT tracer FROM tracer WHERE file_id = ?", (file_id,))
            if row is not None:
                return row[0] or ""
            return ""  # File was measured, but no tracer associated.

    def set_query_context(self, context: str) -> None:
        """Set a context for subsequent querying.

        The next :meth:`lines`, :meth:`arcs`, or :meth:`contexts_by_lineno`
        calls will be limited to only one context.  `context` is a string which
        must match a context exactly.  If it does not, no exception is raised,
        but queries will return no data.

        .. versionadded:: 5.0

        """
        self._start_using()
        with self._connect() as con:
            with con.execute("SELECT id FROM context WHERE context = ?", (context,)) as cur:
                self._query_context_ids = [row[0] for row in cur.fetchall()]

    def set_query_contexts(self, contexts: Sequence[str] | None) -> None:
        """Set a number of contexts for subsequent querying.

        The next :meth:`lines`, :meth:`arcs`, or :meth:`contexts_by_lineno`
        calls will be limited to the specified contexts.  `contexts` is a list
        of Python regular expressions.  Contexts will be matched using
        :func:`re.search <python:re.search>`.  Data will be included in query
        results if they are part of any of the contexts matched.

        .. versionadded:: 5.0

        """
        self._start_using()
        if contexts:
            with self._connect() as con:
                context_clause = " or ".join(["context REGEXP ?"] * len(contexts))
                with con.execute("SELECT id FROM context WHERE " + context_clause, contexts) as cur:
                    self._query_context_ids = [row[0] for row in cur.fetchall()]
        else:
            self._query_context_ids = None

    def lines(self, filename: str) -> list[TLineNo] | None:
        """Get the list of lines executed for a source file.

        If the file was not measured, returns None.  A file might be measured,
        and have no lines executed, in which case an empty list is returned.

        If the file was executed, returns a list of integers, the line numbers
        executed in the file. The list is in no particular order.

        """
        self._start_using()
        if self.has_arcs():
            arcs = self.arcs(filename)
            if arcs is not None:
                all_lines = itertools.chain.from_iterable(arcs)
                return list({l for l in all_lines if l > 0})

        with self._connect() as con:
            file_id = self._file_id(filename)
            if file_id is None:
                return None
            else:
                query = "SELECT numbits FROM line_bits WHERE file_id = ?"
                data = [file_id]
                if self._query_context_ids is not None:
                    ids_array = ", ".join("?" * len(self._query_context_ids))
                    query += " AND context_id IN (" + ids_array + ")"
                    data += self._query_context_ids
                with con.execute(query, data) as cur:
                    bitmaps = list(cur)
                nums = set()
                for row in bitmaps:
                    nums.update(numbits_to_nums(row[0]))
                return list(nums)

    def arcs(self, filename: str) -> list[TArc] | None:
        """Get the list of arcs executed for a file.

        If the file was not measured, returns None.  A file might be measured,
        and have no arcs executed, in which case an empty list is returned.

        If the file was executed, returns a list of 2-tuples of integers. Each
        pair is a starting line number and an ending line number for a
        transition from one line to another. The list is in no particular
        order.

        Negative numbers have special meaning.  If the starting line number is
        -N, it represents an entry to the code object that starts at line N.
        If the ending ling number is -N, it's an exit from the code object that
        starts at line N.

        """
        self._start_using()
        with self._connect() as con:
            file_id = self._file_id(filename)
            if file_id is None:
                return None
            else:
                query = "SELECT DISTINCT fromno, tono FROM arc WHERE file_id = ?"
                data = [file_id]
                if self._query_context_ids is not None:
                    ids_array = ", ".join("?" * len(self._query_context_ids))
                    query += " AND context_id IN (" + ids_array + ")"
                    data += self._query_context_ids
                with con.execute(query, data) as cur:
                    return list(cur)

    def contexts_by_lineno(self, filename: str) -> dict[TLineNo, list[str]]:
        """Get the contexts for each line in a file.

        Returns:
            A dict mapping line numbers to a list of context names.

        .. versionadded:: 5.0

        """
        self._start_using()
        with self._connect() as con:
            file_id = self._file_id(filename)
            if file_id is None:
                return {}

            lineno_contexts_map = collections.defaultdict(set)
            if self.has_arcs():
                query = """
                    SELECT arc.fromno, arc.tono, context.context
                    FROM arc, context
                    WHERE arc.file_id = ? AND arc.context_id = context.id
                """
                data = [file_id]
                if self._query_context_ids is not None:
                    ids_array = ", ".join("?" * len(self._query_context_ids))
                    query += " AND arc.context_id IN (" + ids_array + ")"
                    data += self._query_context_ids
                with con.execute(query, data) as cur:
                    for fromno, tono, context in cur:
                        if fromno > 0:
                            lineno_contexts_map[fromno].add(context)
                        if tono > 0:
                            lineno_contexts_map[tono].add(context)
            else:
                query = """
                    SELECT l.numbits, c.context FROM line_bits l, context c
                    WHERE l.context_id = c.id
                    AND file_id = ?
                """
                data = [file_id]
                if self._query_context_ids is not None:
                    ids_array = ", ".join("?" * len(self._query_context_ids))
                    query += " AND l.context_id IN (" + ids_array + ")"
                    data += self._query_context_ids
                with con.execute(query, data) as cur:
                    for numbits, context in cur:
                        for lineno in numbits_to_nums(numbits):
                            lineno_contexts_map[lineno].add(context)

        return {lineno: list(contexts) for lineno, contexts in lineno_contexts_map.items()}

    @classmethod
    def sys_info(cls) -> list[tuple[str, Any]]:
        """Our information for `Coverage.sys_info`.

        Returns a list of (key, value) pairs.

        """
        with SqliteDb(":memory:", debug=NoDebugging()) as db:
            with db.execute("PRAGMA temp_store") as cur:
                temp_store = [row[0] for row in cur]
            with db.execute("PRAGMA compile_options") as cur:
                copts = [row[0] for row in cur]
            copts = textwrap.wrap(", ".join(copts), width=75)

        return [
            ("sqlite3_sqlite_version", sqlite3.sqlite_version),
            ("sqlite3_temp_store", temp_store),
            ("sqlite3_compile_options", copts),
        ]


def filename_suffix(suffix: str | bool | None) -> str | None:
    """Compute a filename suffix for a data file.

    If `suffix` is a string or None, simply return it. If `suffix` is True,
    then build a suffix incorporating the hostname, process id, and a random
    number.

    Returns a string or None.

    """
    if suffix is True:
        # If data_suffix was a simple true value, then make a suffix with
        # plenty of distinguishing information.  We do this here in
        # `save()` at the last minute so that the pid will be correct even
        # if the process forks.
        die = random.Random(os.urandom(8))
        letters = string.ascii_uppercase + string.ascii_lowercase
        rolls = "".join(die.choice(letters) for _ in range(6))
        suffix = f"{socket.gethostname()}.{os.getpid()}.X{rolls}x"
    elif suffix is False:
        suffix = None
    return suffix