commit dd1a7078203e7a15d0b29da33900c89855b3b7c7
parent 33040296026b8c9901ea52a8628cbebe78b710b5
Author: mtmn <miro@haravara.org>
Date: Fri, 3 Jul 2026 15:29:47 +0200
diggah: use fd for remote search
Diffstat:
| M | diggah/diggah.py | | | 139 | +++++++++++++++++++++++++++++++++++++++++++++++-------------------------------- |
1 file changed, 82 insertions(+), 57 deletions(-)
diff --git a/diggah/diggah.py b/diggah/diggah.py
@@ -3,7 +3,7 @@ import os
import sys
from collections.abc import Iterable
from datetime import datetime, timedelta
-from typing import cast
+from typing import TextIO, cast
import fire
import paramiko
@@ -56,52 +56,58 @@ class FileSystemSearcher:
def find_items(
self,
- start_ts: float | None,
- end_ts: float | None,
+ windows: list[tuple[float | None, float | None]],
include_files: bool = False,
root: str = ".",
- ):
+ ) -> Iterable[tuple[int, str]]:
"""
- Yields paths of items modified between start_ts and end_ts (inclusive).
+ Walks the tree once and yields (window_index, path) for items whose
+ mtime falls within one of the (start_ts, end_ts) windows.
"""
for dirpath, _, filenames in os.walk(root):
- if self._is_modified_in_range(dirpath, start_ts, end_ts):
- yield dirpath
+ index = self._window_index(dirpath, windows)
+ if index is not None:
+ yield index, dirpath
if include_files:
for filename in filenames:
filepath = os.path.join(dirpath, filename)
- if self._is_modified_in_range(filepath, start_ts, end_ts):
- yield filepath
+ index = self._window_index(filepath, windows)
+ if index is not None:
+ yield index, filepath
- def _is_modified_in_range(
- self, path: str, start_ts: float | None, end_ts: float | None
- ) -> bool:
+ def _window_index(
+ self, path: str, windows: list[tuple[float | None, float | None]]
+ ) -> int | None:
try:
mtime = os.path.getmtime(path)
+ except OSError:
+ return None
+ for index, (start_ts, end_ts) in enumerate(windows):
if start_ts is not None and mtime <= start_ts:
- return False
+ continue
if end_ts is not None and mtime >= end_ts:
- return False
- return True
- except OSError:
- return False
+ continue
+ return index
+ return None
class RemoteExecutor:
"""Handles executing the script on a remote host via SSH using Paramiko."""
+ SENTINEL = "::diggah-next-window::"
+
def execute(
self,
host_str: str,
path: str,
- start_dt: datetime | None,
- end_dt: datetime | None,
+ windows: list[tuple[datetime | None, datetime | None]],
include_files: bool,
port: int | None = None,
- ) -> Iterable[str]:
+ ) -> Iterable[tuple[int, str]]:
"""
- Executes a find command on the remote host and yields the results.
+ Executes a single fd command chain on the remote host over one SSH
+ connection and yields (window_index, path) results.
"""
username = None
hostname = host_str
@@ -135,20 +141,25 @@ class RemoteExecutor:
port = int(user_config.get("port", 22))
key_filename = user_config.get("identityfile")
- cmd_parts = ["find", path]
+ commands: list[str] = []
+ for start_dt, end_dt in windows:
+ cmd_parts = ["fd", "--hidden", "--no-ignore"]
+
+ if not include_files:
+ cmd_parts.extend(["--type", "d"])
- if not include_files:
- cmd_parts.extend(["-type", "d"])
+ if start_dt:
+ start_str = start_dt.strftime("%Y-%m-%d %H:%M:%S")
+ cmd_parts.extend(["--changed-after", f"'{start_str}'"])
- if start_dt:
- start_str = start_dt.strftime("%Y-%m-%d %H:%M:%S")
- cmd_parts.extend(["-newermt", f"'{start_str}'"])
+ if end_dt:
+ end_str = end_dt.strftime("%Y-%m-%d %H:%M:%S")
+ cmd_parts.extend(["--changed-before", f"'{end_str}'"])
- if end_dt:
- end_str = end_dt.strftime("%Y-%m-%d %H:%M:%S")
- cmd_parts.extend(["!", "-newermt", f"'{end_str}'"])
+ cmd_parts.extend([".", path])
+ commands.append(" ".join(cmd_parts))
- cmd = " ".join(cmd_parts)
+ cmd = f"; echo '{self.SENTINEL}'; ".join(commands)
client = paramiko.SSHClient()
client.load_system_host_keys()
@@ -171,10 +182,13 @@ class RemoteExecutor:
stdin.close()
# Read stderr first/async or just handle stdout for now.
+ index = 0
for line in cast(Iterable[str], stdout):
line_str = str(line).strip()
- if line_str:
- yield line_str
+ if line_str == self.SENTINEL:
+ index += 1
+ elif line_str:
+ yield index, line_str
for line in cast(Iterable[str], stderr):
print(str(line), end="", file=sys.stderr)
@@ -245,15 +259,14 @@ class Diggah:
fire.Fire(self.search, command=["--help"], name="diggah search")
return
- for start_dt, end_dt, outfile in ranges:
- if dry_run:
+ if dry_run:
+ for start_dt, end_dt, outfile in ranges:
self._print_dry_run(
start_dt, end_dt, path, files, host, port, relative, output, outfile
)
- else:
- self._run_search(
- start_dt, end_dt, path, files, host, port, relative, outfile
- )
+ return
+
+ self._run_search(ranges, path, files, host, port, relative)
def _determine_search_ranges(
self,
@@ -361,7 +374,7 @@ class Diggah:
print(f"Host: {host}")
if port:
print(f"Port: {port}")
- print("Mode: Remote (find command)")
+ print("Mode: Remote (fd command)")
else:
print("Mode: Local (python walk)")
if relative:
@@ -371,51 +384,63 @@ class Diggah:
def _run_search(
self,
- start_dt: datetime | None,
- end_dt: datetime | None,
+ ranges: list[tuple[datetime | None, datetime | None, str | None]],
path: str,
include_files: bool,
host: str | None,
port: int | None,
relative: bool,
- outfile: str | None,
):
- results: Iterable[str]
+ results: Iterable[tuple[int, str]]
if host:
- results = RemoteExecutor().execute(
- host, path, start_dt, end_dt, include_files, port
- )
+ windows = [(start_dt, end_dt) for start_dt, end_dt, _ in ranges]
+ results = RemoteExecutor().execute(host, path, windows, include_files, port)
else:
- start_ts = start_dt.timestamp() if start_dt else None
- end_ts = end_dt.timestamp() if end_dt else None
+ ts_windows = [
+ (
+ start_dt.timestamp() if start_dt else None,
+ end_dt.timestamp() if end_dt else None,
+ )
+ for start_dt, end_dt, _ in ranges
+ ]
results = FileSystemSearcher().find_items(
- start_ts,
- end_ts,
+ ts_windows,
include_files=include_files,
root=path,
)
- f = None
- if outfile:
+ sinks: list[TextIO | None] = []
+ failed: set[int] = set()
+ for index, (_, _, outfile) in enumerate(ranges):
+ if not outfile:
+ sinks.append(None)
+ continue
try:
f = open(outfile, "w")
print(f"{outfile}")
+ sinks.append(f)
except IOError as e:
print(f"opening output file: {e} failed", file=sys.stderr)
- return
+ sinks.append(None)
+ failed.add(index)
try:
- for item in results:
+ for index, item in results:
+ if index in failed:
+ continue
+
if relative:
item = os.path.relpath(item, start=path)
+ f = sinks[index]
if f:
_ = f.write(item + "\n")
else:
print(item)
finally:
- if f:
- f.close()
+ for f in sinks:
+ if f:
+ f.close()
def main():
