diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba1c6b8..309370b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,5 +7,8 @@ version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests + # Updates belong on dev: main is the released branch and merging into it + # publishes, so a PR opened against main just sits there. + target-branch: "dev" schedule: interval: "daily" diff --git a/.gitignore b/.gitignore index c9b638b..831dcb6 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ credentials.json **/credentials.json .claude/ + +# Local Codacy / Sonar scratch output +.codacy_tmp/ diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 26d3352..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml diff --git a/.idea/FileAutomation.iml b/.idea/FileAutomation.iml deleted file mode 100644 index 91f2d42..0000000 --- a/.idea/FileAutomation.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 2cefeb6..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 705b883..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 8fd3887..787c2ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,69 +8,41 @@ Automation-first Python library for local file / directory / zip operations, HTT ``` automation_file/ -├── __init__.py # Public API facade (every name users import) -├── __main__.py # CLI entry (argparse dispatcher, subcommands + legacy flags) -├── exceptions.py # Exception hierarchy (FileAutomationException base) -├── logging_config.py # file_automation_logger (file + stderr handlers) -├── core/ -│ ├── action_registry.py # ActionRegistry — name -> callable (Registry + Command) -│ ├── action_executor.py # ActionExecutor — runs JSON action lists (Facade + Template Method) -│ ├── callback_executor.py # CallbackExecutor — trigger then callback composition -│ ├── package_loader.py # PackageLoader — dynamically registers package members -│ ├── json_store.py # Thread-safe read/write of JSON action files -│ ├── retry.py # retry_on_transient — capped exponential back-off decorator -│ └── quota.py # Quota — size + time budget guards -├── local/ # Strategy modules — each file is a batch of pure operations -│ ├── file_ops.py -│ ├── dir_ops.py -│ ├── zip_ops.py -│ └── safe_paths.py # safe_join / is_within — path traversal guard -├── remote/ -│ ├── url_validator.py # SSRF guard for outbound URLs -│ ├── http_download.py # SSRF-validated HTTP download with size/timeout caps + retry -│ ├── google_drive/ -│ │ ├── client.py # GoogleDriveClient (Singleton Facade) -│ │ ├── delete_ops.py -│ │ ├── download_ops.py -│ │ ├── folder_ops.py -│ │ ├── search_ops.py -│ │ ├── share_ops.py -│ │ └── upload_ops.py -│ ├── s3/ # S3 (boto3) — auto-registered in build_default_registry() -│ │ ├── client.py # S3Client -│ │ ├── upload_ops.py -│ │ ├── download_ops.py -│ │ ├── delete_ops.py -│ │ └── list_ops.py -│ ├── azure_blob/ # Azure Blob — auto-registered in build_default_registry() -│ │ └── {client,upload,download,delete,list}_ops.py -│ ├── dropbox_api/ # Dropbox — auto-registered in build_default_registry() -│ │ └── {client,upload,download,delete,list}_ops.py -│ └── sftp/ # SFTP (paramiko + RejectPolicy) — auto-registered in build_default_registry() -│ └── {client,upload,download,delete,list}_ops.py -├── server/ -│ ├── tcp_server.py # Loopback-only TCP server executing JSON actions (optional shared-secret auth) -│ └── http_server.py # Loopback-only HTTP server (POST /actions, optional Bearer auth) -├── project/ -│ ├── project_builder.py # ProjectBuilder (Builder pattern) -│ └── templates.py # Scaffolding templates -├── ui/ # PySide6 GUI (required dep) -│ ├── launcher.py # launch_ui(argv) — boots QApplication + MainWindow -│ ├── main_window.py # MainWindow — tabbed control surface over every feature -│ ├── worker.py # ActionWorker(QRunnable) + _WorkerSignals -│ ├── log_widget.py # LogPanel — timestamped, read-only log stream -│ └── tabs/ # One tab per domain: local / http / drive / s3 / -│ # azure / dropbox / sftp / -│ # JSON actions / servers -└── utils/ - └── file_discovery.py # Recursive file listing by extension +├── __init__.py # Public API facade (__all__); launch_ui is loaded lazily via __getattr__ +├── __main__.py # CLI entry: subcommands plus the legacy -e/-d/-c/--execute_str flags +├── exceptions.py # FileAutomationException hierarchy +├── logging_config.py # file_automation_logger (file + stderr handlers) +├── core/ # Engine: action_registry (ActionRegistry, build_default_registry), action_executor +│ # (shared `executor`), callback_executor, package_loader, plugins, dag_executor, +│ # action_queue, json_store, substitution; cross-cutting helpers: retry, quota, +│ # rate_limit, circuit_breaker, file_lock, sqlite_lock, checksum, manifest, crypto, +│ # secrets, config, config_watcher, audit, metrics, tracing, progress, fim, content_store +├── local/ # Strategy modules: file/dir/zip/tar/archive ops, sync, diff, text/JSON/data edits, +│ # templates, versioning, trash, shell_ops (argv-only subprocess), conditional; +│ # safe_paths.py guards against path traversal +├── remote/ # url_validator (SSRF guard), http_download, cross_backend, fsspec_bridge, and one +│ # subpackage per backend: google_drive, s3, azure_blob, dropbox_api, sftp, ftp, +│ # onedrive, box (client.py + *_ops.py + register__ops); smb and webdav +│ # have a client only +├── server/ # tcp_server, http_server, mcp_server (MCP over stdio), web_ui, metrics_server, +│ # action_acl (ActionACL), network_guards (ensure_loopback) +├── client/ # HTTPActionClient for the HTTP action server +├── trigger/, scheduler/, notify/ # watchdog file triggers, cron scheduler, notification sinks; +│ # each registers its own FA_* ops +├── project/ # ProjectBuilder, create_project_dir +├── ui/ # PySide6 GUI: launcher.launch_ui, main_window.MainWindow, worker.ActionWorker, +│ # log_widget.LogPanel, tabs/ (home, local, http, JSON editor, servers, scheduler, +│ # trigger, progress; the cloud backends are panels grouped under transfer_tab) +└── utils/ # file discovery, fast find, grep, duplicate finder, backup rotation ``` +`architecture.md` §2 carries the same map with one row per directory; keep the two in step. + **Key design patterns in use:** - **Facade**: `automation_file/__init__.py` re-exports every supported name (`execute_action`, `driver_instance`, `start_autocontrol_socket_server`, …). - **Registry + Command**: `ActionRegistry` maps action name → callable. JSON action lists are command objects (`[name, kwargs]` / `[name, [args]]` / `[name]`) dispatched through the registry. - **Template Method**: `ActionExecutor._execute_event` defines the single-action lifecycle (resolve → call → wrap result); `execute_action` is the outer iteration template. -- **Strategy**: Each `local/*_ops.py` and `remote/google_drive/*_ops.py` module is an independent strategy that plugs into the registry. +- **Strategy**: Each `local/*_ops.py` and `remote//*_ops.py` module is an independent strategy that plugs into the registry. - **Singleton (module-level)**: `driver_instance`, `executor`, `callback_executor`, `package_manager` are shared instances wired in `__init__.py` so `callback_executor.registry is executor.registry`. - **Builder**: `ProjectBuilder` assembles the `keyword/` + `executor/` skeleton. @@ -85,7 +57,7 @@ automation_file/ - `MainWindow` — PySide6 tabbed control surface (`ui/main_window.py`). Nine tabs — Local, HTTP, Google Drive, S3, Azure Blob, Dropbox, SFTP, JSON actions, Servers — share a `LogPanel` and dispatch work through `ActionWorker(QRunnable)` on the global `QThreadPool`. - `launch_ui(argv=None)` — boots / reuses a `QApplication`, shows `MainWindow`, and returns the exec code. Exposed lazily on the facade via `__getattr__` so the Qt runtime isn't paid for by non-UI importers. - `TCPActionServer` — threaded TCP server that deserialises a JSON action list per connection. Defaults to loopback; optional `shared_secret` enforces `AUTH \n` prefix. -- `HTTPActionServer` — `ThreadingHTTPServer` exposing `POST /actions`. Defaults to loopback; optional `shared_secret` enforces `Authorization: Bearer `. +- `HTTPActionServer` — `ThreadingHTTPServer` exposing `POST /actions` plus `GET /healthz`, `/readyz`, `/openapi.json` and `/progress`. Defaults to loopback; optional `shared_secret` enforces `Authorization: Bearer `. - `Quota` — frozen dataclass capping bytes and wall-clock seconds per action or block (`check_size`, `time_budget` context manager, `wraps` decorator). `0` disables each cap. - `retry_on_transient(max_attempts, backoff_base, backoff_cap, retriable)` — decorator that retries with capped exponential back-off and raises `RetryExhaustedException` chained to the last error. - `safe_join(root, user_path)` / `is_within(root, path)` — path traversal guard; `safe_join` raises `PathTraversalException` when the resolved path escapes `root`. @@ -96,7 +68,7 @@ automation_file/ - `dev` branch: development, publishes `automation_file_dev` to PyPI (version in `dev.toml`). - Keep `dependencies` and `[project.optional-dependencies]` (`dev`) in sync across both TOMLs. Backends (`boto3`, `azure-storage-blob`, `dropbox`, `paramiko`) and `PySide6` are first-class runtime deps — do not move them back under extras. - **Version bumping is automatic.** A dedicated publish workflow bumps the patch in both `stable.toml` and `dev.toml`, builds, uploads to PyPI, then commits the bump back to `main` tagged as `vX.Y.Z`. Do not hand-bump before merging to `main`. The next publish run is skipped via a commit-message guard (`chore: bump version`), so the bump itself never re-triggers publishing. -- CI: GitHub Actions (Windows, Python 3.10 / 3.11 / 3.12) — one matrix workflow per branch: `.github/workflows/ci-dev.yml`, `.github/workflows/ci-stable.yml`. +- CI: GitHub Actions — a `lint` job on Ubuntu (Python 3.12), then `pytest` on Windows across Python 3.10 / 3.11 / 3.12 / 3.13 / 3.14. One workflow per branch: `.github/workflows/ci-dev.yml`, `.github/workflows/ci-stable.yml`. - CI steps: `lint` (ruff check + ruff format --check + mypy) → `pytest` with coverage → uploads `coverage.xml` as an artifact. - Publishing lives in a separate workflow (`.github/workflows/publish.yml`) that runs on push to `main`: bumps both TOMLs, copies `stable.toml` to `pyproject.toml`, builds the sdist + wheel, `twine upload` via `PYPI_API_TOKEN`, then commits + tags + pushes and creates `gh release create v --generate-notes`. - `pre-commit` is configured (`.pre-commit-config.yaml`): trailing-whitespace, eof-fixer, check-yaml, check-toml, check-added-large-files, ruff, ruff-format, mypy. Install with `pre-commit install` after cloning. @@ -163,7 +135,7 @@ All code must follow secure-by-default principles. Review every change against t ### HTTP server - `HTTPActionServer` / `start_http_action_server` mirror the TCP server's posture: loopback-only by default, `allow_non_loopback=True` required to bind elsewhere, optional `shared_secret` enforced as `Authorization: Bearer ` using `hmac.compare_digest`. -- Only `POST /actions` is handled. Request body capped at 1 MB — do not raise without also switching to a streaming parser. +- `POST /actions` is the only endpoint that runs anything; the `GET` routes (`/healthz`, `/readyz`, `/openapi.json`, `/progress`) only report. Request body capped at 1 MB — do not raise without also switching to a streaming parser. - Responses are JSON. Auth failures return `401`; malformed JSON returns `400`; unknown paths return `404`. ### Path traversal @@ -267,6 +239,18 @@ All code must satisfy common static-analysis rules. Review every change against - Before committing any non-trivial change, run `ruff check automation_file/ tests/` locally. - When adding a `# noqa: RULE`, justify it in the comment — never blanket-disable. +## Stage commits, `progress.md`, `docs/updates/` and `architecture.md` + +Workspace rule shared by every repository under `D:\Codes` (full text: `D:\Codes\CLAUDE.md`). + +- **Commit at every stage.** A stage is the smallest piece of work that leaves the repository consistent and passes this project's checks (definition of done, tests, lint): one finished `progress.md` item, or one self-contained step of a larger one. Commit it before starting the next stage, before switching to another repository, and before the session ends. Do not leave work uncommitted across sessions; if a stage cannot be finished, commit the consistent part and record the rest in `progress.md`. + - Stage only the files that stage touched (`git add `, never `git add -A`), follow this file's commit-message rules, and never add AI attribution. + - Committing is not pushing: push or open a PR only as this project's branch flow says or when asked. +- **`progress.md`** (repository root, tracked) holds outstanding work only: no finished items, no history, no rules. +- **`docs/updates/`** records finished work: one batch file per month (`YYYY-MM.md`), one entry per piece of work headed `## U-YYYYMMDD-NN · date · title · #tags`, and an index with query commands in `docs/updates/README.md`. When a `progress.md` item is done, delete it and add a `#done` entry plus its index row in the same commit. +- **`architecture.md`** (repository root) is the short architecture overview: layers, entry points, main flows, extension points, cross-project boundaries. Update it in the same commit whenever a change alters any of those. +- **Cross-project contracts** are listed in `architecture.md` §6: what other repositories rely on here (CLI flags, import paths, constructor arguments, file layouts) and what this repository relies on elsewhere. No test here protects them, so never rename or remove one without changing its consumers in the same round, and update §6 whenever a contract is added or changes. + ## Commit & PR rules - Commit messages: short imperative sentence (e.g., "Fix rename_file overwrite bug", "Update stable version"). diff --git a/architecture.md b/architecture.md new file mode 100644 index 0000000..03ef11e --- /dev/null +++ b/architecture.md @@ -0,0 +1,160 @@ +# FileAutomation Architecture + +> Short overview for people and agents. +> Last verified: 2026-09-22 against `2e6c0ec` on `dev`. + +## 1. Purpose + +FileAutomation (`automation_file`) is an automation-first library for local file, directory and +archive operations, SSRF-checked HTTP downloads, and remote storage (Google Drive, S3, Azure Blob, +Dropbox, SFTP, FTP, OneDrive, Box, plus SMB and WebDAV clients). Every operation is an `FA_*` +command in one `ActionRegistry`, so JSON action lists run the same way in-process, from files, from +the CLI, over loopback TCP or HTTP servers, as MCP tools, or from the PySide6 GUI. + +## 2. Layers and directories + +| Path | Responsibility | +| --- | --- | +| `automation_file/__init__.py` | Public facade (`__all__`). Wires the shared `executor`, `callback_executor` and `package_manager` over one registry. `launch_ui` is loaded lazily through `__getattr__` | +| `automation_file/__main__.py` | CLI: legacy flags plus subcommands | +| `automation_file/core/` | Engine: `action_registry.py` (`ActionRegistry`, `build_default_registry`), `action_executor.py` (`ActionExecutor`, shared `executor`), `callback_executor.py`, `package_loader.py`, `plugins.py`, `dag_executor.py`, `action_queue.py`, `json_store.py`, `substitution.py`. Also cross-cutting helpers: `retry`, `quota`, `rate_limit`, `circuit_breaker`, `file_lock`, `sqlite_lock`, `checksum`, `manifest`, `crypto`, `secrets`, `config`, `config_watcher`, `audit`, `metrics`, `tracing`, `progress`, `fim`, `content_store` | +| `automation_file/local/` | Local strategy modules: file, dir, zip, tar and archive ops, sync, diff, text/JSON/data edits, templates, versioning, trash, `shell_ops` (argv-only subprocess), conditional branches. `safe_paths.py` guards against path traversal | +| `automation_file/remote/` | `url_validator.py` (SSRF guard), `http_download.py`, `cross_backend.py`, `fsspec_bridge.py`. One subpackage per backend: `google_drive/`, `s3/`, `azure_blob/`, `dropbox_api/`, `sftp/`, `ftp/`, `onedrive/`, `box/`, each with `client.py`, `*_ops.py` and `register__ops`. `smb/` and `webdav/` have a client only | +| `automation_file/server/` | `tcp_server.py`, `http_server.py`, `mcp_server.py`, `web_ui.py`, `metrics_server.py`, `action_acl.py` (`ActionACL`), `network_guards.py` (`ensure_loopback`) | +| `automation_file/client/` | `HTTPActionClient` for the HTTP action server | +| `automation_file/trigger/`, `scheduler/`, `notify/` | Watchdog file triggers, cron scheduler, notification sinks. Each registers its own `FA_*` ops | +| `automation_file/project/` | `ProjectBuilder`, `create_project_dir` | +| `automation_file/ui/` | PySide6 GUI: `launcher.launch_ui`, `main_window.MainWindow`, `worker.ActionWorker`, `log_widget.LogPanel`, `tabs/` (backend panels are grouped under `TransferTab`) | +| `automation_file/utils/` | File discovery, fast find, grep, duplicate finder, backup rotation | +| `automation_file/exceptions.py`, `logging_config.py` | `FileAutomationException` hierarchy; `file_automation_logger` (INFO+ to stderr, DEBUG+ to `$FILE_AUTOMATION_LOG_FILE` or `~/.automation_file/logs/FileAutomation.log`, opened on first use) | +| `stable.toml`, `dev.toml` | Packaging for `automation_file` and `automation_file_dev`. No `pyproject.toml` is committed; CI and publish copy one of these TOMLs into place | +| `main_ui.py` | Development shortcut for `launch_ui()` | +| `tests/`, `docs/`, `examples/mcp/` | pytest suite (fixtures in `tests/conftest.py`); Sphinx docs; MCP host configuration example | + +## 3. Entry points and public interfaces + +- **Python facade** (`import automation_file`): `execute_action`, `execute_files`, + `execute_action_parallel`, `validate_action`, `execute_action_dag`, `add_command_to_executor`, + `executor`, `callback_executor`, `package_manager`, `ActionRegistry`, `build_default_registry`, + `driver_instance` (Google Drive), `start_autocontrol_socket_server`, `start_http_action_server`, + `HTTPActionClient`, `MCPServer`, `create_project_dir`, `launch_ui` (lazy). +- **Action format**: an action is `[name]`, `[name, {kwargs}]` or `[name, [args]]`. A file holds a + list of actions or `{"auto_control": [...]}`. +- **CLI** (`python -m automation_file`; no console script for it): + - legacy flags `-e/--execute_file`, `-d/--execute_dir`, `-c/--create_project` and `--execute_str`. + `_execute_str` decodes a second time when the first `json.loads` yields a string; + - subcommands `zip`, `unzip`, `download`, `create-file`, `server`, `http-server`, `ui`, `mcp`, `drive-upload`. +- **MCP**: `automation_file_mcp` (`automation_file.server.mcp_server:_cli`) or + `python -m automation_file mcp [--allowed-actions ...]`. It is a standard-library JSON-RPC stdio + server whose tools come from the registry (`tools_from_registry`). +- **TCP server**: `start_autocontrol_socket_server(host="localhost", port=9943, allow_non_loopback=False, shared_secret=None, action_acl=None)` + returns a `TCPActionServer`. With a secret, clients prefix the payload with `AUTH \n`. + `quit_server` shuts it down. Replies end with `Return_Data_Over_JE\n`. +- **HTTP server**: `start_http_action_server` returns an `HTTPActionServer` (default `127.0.0.1:9944`) + with `POST /actions` and `GET /healthz`, `/readyz`, `/openapi.json`, `/progress`. Auth is an + optional `Bearer` token. +- **Other servers**: `start_web_ui` (default port 9955) and `start_metrics_server` (`/metrics`, + default port 9945). +- **GUI**: `launch_ui()`, `python -m automation_file ui` or `python main_ui.py`. +- **Plugins**: third-party packages register actions through the entry-point group `automation_file.actions`. + +## 4. Main flows + +**Action list → result** + +``` +JSON file / --execute_str / Python → ActionExecutor.execute_action(list|dict, validate_first, dry_run, substitute) + → _coerce (list or {"auto_control": [...]}) → _execute_event → registry.resolve(name) + → FA_* callable in local/ | remote/ | utils/ | core/ (inside tracing.action_span) + → {"execute: ": return value | repr(error)} (one failure never aborts the batch) +``` + +**Remote transports** + +``` +start_autocontrol_socket_server / start_http_action_server → ensure_loopback (unless allow_non_loopback=True) + → TCP (AUTH + JSON) | HTTP POST /actions (Bearer) → ActionACL check → shared executor +MCP host → automation_file_mcp (stdio JSON-RPC) → tools/call → MCPServer registry (optionally --allowed-actions) +``` + +**Registry construction** + +``` +ActionExecutor() → build_default_registry(): local + http + utils + drive commands + → _register_cloud_backends (register__ops) → trigger / scheduler / progress / notify ops + → _load_plugins (entry points; may override built-ins) + → executor adds FA_execute_action, FA_execute_files, FA_execute_action_parallel, FA_validate +``` + +## 5. Extension points + +- **New local or utility action**: function in `local/_ops.py` (or `utils/`, `core/`) using + `from __future__ import annotations`, `FileAutomationException` subclasses and `file_automation_logger` + → `"FA_"` in `_local_commands()` or `_utils_commands()` (`core/action_registry.py`) → export + from `automation_file/__init__.py` and `__all__` → `tests/test_.py` (plus `tests/test_facade.py` + when exported). +- **New remote backend**, in this order: + 1. `remote//` with `client.py` (module singleton `_instance` with `later_init`, + plus `close` where relevant), the `*_ops.py` modules, and `register__ops(registry)` in `__init__.py`. + 2. Call it from `_register_cloud_backends`. + 3. Add the SDK to `dependencies` in both `stable.toml` and `dev.toml`, then add facade exports. + 4. Add `ui/tabs/_tab.py` and wire it into `ui/tabs/transfer_tab.py`. + 5. Add tests; paths that need the network are not exercised in CI. +- **Outbound HTTP**: always call `validate_http_url` (`remote/url_validator.py`) first. +- **Plugins**: an entry point in the group `automation_file.actions` (`core/plugins.py`), or + `add_command_to_executor({...})` at runtime. `package_manager.add_package_to_executor` registers a + package's members as `_`. +- **CLI subcommand**: `_cmd_` registered in an `_add_*_commands` helper (`__main__.py`). Keep the + legacy flags and the double decode. +- **New server surface**: reuse `ensure_loopback`, `shared_secret` with `hmac.compare_digest`, and `ActionACL`. + +## 6. Cross-project boundaries + +- **PyBreeze (subprocess)** runs `python -m automation_file --execute_str ` or `--execute_file ` + (`PyBreeze/pybreeze/extend/process_executor/python_task_process_manager.py`; the package name is in + `.../process_executor/file_automation/file_automation_process.py`). PyBreeze double-encodes the JSON + on Windows, so `_execute_str`'s `isinstance`-guarded second decode and the legacy flag names are a + contract, guarded by `tests/test_legacy_cli_contract.py`. PyBreeze also declares `automation-file` as + a dependency. +- **TestPioneer** imports `download_file` and `unzip_all` from the facade in-process + (`test_pioneer/executor/file/file_processing.py`). Its `parallel_run` does not spawn this package. +- **Names inherited from AutoControl**: the TCP starter is still called `start_autocontrol_socket_server` + and the action-dict key is `auto_control`. MailThunder has renamed both (`start_mail_thunder_socket_server`, `mail_thunder` key) and keeps the old names as deprecated aliases. MailThunder's + socket-server default is 9942, so it runs next to this package's servers (TCP 9943, HTTP 9944, + metrics 9945) on their defaults. +- **Wire format**: TCP replies end with the same `Return_Data_Over_JE` terminator as the sibling servers. +- **Builtins policy**: the default registry contains no Python builtins; only `PackageLoader` can add + them. In the siblings, APITestka uses an explicit allowlist, LoadDensity a `_UNSAFE_BUILTINS` + blacklist, and MailThunder registers every builtin (known gap). + +## 7. Design constraints + +- Only the three action shapes in §3. Extend through the registry, not by subclassing the executor. + Python 3.10+, `X | Y` unions, `from __future__ import annotations` (CLAUDE.md § Conventions). +- Exceptions derive from `FileAutomationException`. Log through `file_automation_logger`; no + `print()` diagnostics and no runtime `assert` (§ Conventions; § Code quality › Logging, printing, assertions). +- Every user-supplied URL goes through `validate_http_url`. Never `verify=False`. Downloads have size + and time caps and do not follow redirects (§ Security › Network requests (SSRF prevention) / (TLS)). +- Servers bind loopback unless `allow_non_loopback=True`; secrets are compared with `hmac.compare_digest`; + TCP reads one `recv(8192)` payload; HTTP bodies are capped at 1 MB (§ Security › TCP server; › HTTP server). +- Resolve user paths through `safe_join` / `is_within` (§ Security › Path traversal). SFTP keeps + `paramiko.RejectPolicy()` (§ Security › SFTP host verification). +- `retry_on_transient` retries only the listed exception types (§ Security › Reliability (retry / quota)). + `PackageLoader` is eval-grade; never expose it remotely (§ Security › Plugin / package loading). +- No `shell=True`; subprocesses use argument lists and a timeout (§ Security › General rules; › Subprocess execution). +- Backends and PySide6 are first-class runtime dependencies. Keep `stable.toml` and `dev.toml` + dependencies in sync, and let the publish workflow bump versions (§ Branching & CI). +- Limits: cyclomatic complexity ≤ 15 (hard cap 20), cognitive complexity ≤ 15, functions ≤ 75 lines, + ≤ 7 parameters, nesting ≤ 4, files ≤ 1000 lines (§ Code quality › Complexity & size). +- Run `ruff check`, `ruff format --check`, `mypy` and `pytest` before committing (§ Development). + Development PRs target `dev`; stable PRs target `main` (§ Commit & PR rules). + +## 8. When to update this file + +- A top-level subpackage, backend or server module is added, removed or renamed. +- CLI flags, subcommands, `[project.scripts]` in the TOMLs, or the entry-point group change. +- The action format, the `auto_control` key, the registry build order, or plugin override semantics change. +- Server defaults (host, port, auth, ACL, terminator) or HTTP routes change. +- A §6 contract changes: PyBreeze invocation, the Windows double decode, the facade names TestPioneer uses. +- A CLAUDE.md section referenced in §7 is renamed or its rule changes. +- Refresh the "Last verified" line whenever this file is re-checked against HEAD. diff --git a/automation_file/core/action_registry.py b/automation_file/core/action_registry.py index 2cd75e1..c474890 100644 --- a/automation_file/core/action_registry.py +++ b/automation_file/core/action_registry.py @@ -271,7 +271,9 @@ def build_default_registry() -> ActionRegistry: _register_progress_ops(registry) _register_notify_ops(registry) _load_plugins(registry) - file_automation_logger.info( + # DEBUG, not INFO: this runs at import, and INFO is mirrored to stderr, so every import -- + # `python -m automation_file --help` included -- printed it. + file_automation_logger.debug( "action_registry: built default registry with %d commands", len(registry) ) return registry diff --git a/automation_file/logging_config.py b/automation_file/logging_config.py index 2b30e0a..577fad5 100644 --- a/automation_file/logging_config.py +++ b/automation_file/logging_config.py @@ -1,20 +1,82 @@ """Module-level logger for automation_file. -A single :data:`file_automation_logger` is exposed. It writes to -``FileAutomation.log`` in append mode and mirrors every record to stderr via a -custom handler. The handler list is rebuilt only once, even if the module is -reloaded, so tests can import this safely. +A single :data:`file_automation_logger` is exposed. It mirrors INFO+ to stderr +and writes DEBUG+ to ``~/.automation_file/logs/FileAutomation.log`` unless +``FILE_AUTOMATION_LOG_FILE`` names another path (a relative one resolves against +the cwd at import time; ``os.devnull`` turns the file off). The handler list is +rebuilt only once, even if the module is reloaded, so tests can import this safely. + +The file used to be ``FileAutomation.log`` in the working directory, opened at +import, so every process that imported the package (PyBreeze, TestPioneer, test +runs) left one wherever it started. It is now opened on the first record, so +importing writes nothing; every process on the account appends to it with its +process id on each line, and it is rotated only when a process opens it, since +Windows cannot rename a file another process holds open. """ from __future__ import annotations import logging +import os import sys +import warnings +from logging.handlers import RotatingFileHandler +from pathlib import Path -_LOG_FORMAT = "%(asctime)s | %(name)s | %(levelname)s | %(message)s" -_LOG_FILENAME = "FileAutomation.log" +_LOG_FORMAT = "%(asctime)s | %(process)d | %(name)s | %(levelname)s | %(message)s" _LOGGER_NAME = "automation_file" +#: Environment variable that overrides where the log file is written. +LOG_FILE_ENV = "FILE_AUTOMATION_LOG_FILE" + +#: A file past this size is moved to ``.1`` when a process opens it. +ROTATE_AT_BYTES = 10 * 1024 * 1024 + + +def default_log_file() -> Path: + """Return the log file path: ``$FILE_AUTOMATION_LOG_FILE``, else the home-directory default.""" + configured = os.environ.get(LOG_FILE_ENV, "").strip() + if configured: + return Path(configured).expanduser() + return Path.home() / ".automation_file" / "logs" / "FileAutomation.log" + + +def _rotate_if_large(path: Path, limit: int) -> None: + """Move ``path`` to ``.1`` past ``limit`` bytes; best effort while it is held.""" + try: + if limit <= 0 or not path.is_file() or path.stat().st_size <= limit: + return + os.replace(path, path.with_name(path.name + ".1")) + except OSError: + return + + +class FileAutomationFileHandler(RotatingFileHandler): + """Append-mode UTF-8 file handler. + + A file that cannot be opened becomes ``os.devnull`` with one warning. + """ + + def __init__(self, filename: str, delay: bool = True) -> None: + super().__init__( + filename=filename, mode="a", encoding="utf-8", errors="backslashreplace", delay=delay + ) + + def _open(self): + path = Path(self.baseFilename) + try: + path.parent.mkdir(parents=True, exist_ok=True) + _rotate_if_large(path, ROTATE_AT_BYTES) + return super()._open() + except OSError as error: + warnings.warn( + f"FileAutomation log file {path} unavailable, file logging off: {error!r}", + RuntimeWarning, + stacklevel=2, + ) + # The handler owns this stream and closes it in close(). + return open(os.devnull, self.mode, encoding=self.encoding, errors=self.errors) # pylint: disable=consider-using-with + class _StderrHandler(logging.Handler): """Mirror log records to stderr so scripts see progress without enabling root.""" @@ -35,7 +97,7 @@ def _build_logger() -> logging.Logger: formatter = logging.Formatter(_LOG_FORMAT) - file_handler = logging.FileHandler(filename=_LOG_FILENAME, mode="a", encoding="utf-8") + file_handler = FileAutomationFileHandler(str(default_log_file())) file_handler.setFormatter(formatter) file_handler.setLevel(logging.DEBUG) logger.addHandler(file_handler) diff --git a/automation_file/remote/box/client.py b/automation_file/remote/box/client.py index ea63b36..86c2f62 100644 --- a/automation_file/remote/box/client.py +++ b/automation_file/remote/box/client.py @@ -1,10 +1,11 @@ -"""Box client (Singleton Facade) backed by the ``boxsdk`` library. +"""Box client (Singleton Facade) backed by ``box_sdk_gen``. -Box's OAuth2 flow is authorization-code based (not device-code), so the -caller is expected to obtain an access token via their app registration -and hand it in to :meth:`later_init`. Matches the Dropbox backend's -contract — automation workflows typically receive the token from a -secrets manager rather than prompting interactively. +``box_sdk_gen`` is the module shipped by the ``boxsdk`` distribution from 10.0 on; the legacy +``boxsdk.Client`` API it replaced is no longer developed. Box's OAuth2 flow is authorization-code +based (not device-code), so the caller is expected to obtain an access token via their app +registration and hand it in to :meth:`later_init`. Matches the Dropbox backend's contract — +automation workflows typically receive the token from a secrets manager rather than prompting +interactively. """ from __future__ import annotations @@ -15,18 +16,19 @@ from automation_file.logging_config import file_automation_logger -def _import_boxsdk() -> Any: +def import_box_sdk_gen() -> Any: + """Import ``box_sdk_gen``, raising :class:`BoxException` when it is missing.""" try: - import boxsdk + import box_sdk_gen except ImportError as error: raise BoxException( - "boxsdk import failed — reinstall `automation_file` to restore the Box backend" + "box_sdk_gen import failed — install `boxsdk>=10` to restore the Box backend" ) from error - return boxsdk + return box_sdk_gen class BoxClient: - """Lazy wrapper around :class:`boxsdk.Client`.""" + """Lazy wrapper around :class:`box_sdk_gen.BoxClient`.""" def __init__(self) -> None: self.client: Any = None @@ -38,25 +40,27 @@ def later_init( client_id: str = "", client_secret: str = "", ) -> Any: - """Build a :class:`boxsdk.Client` from an OAuth2 access token. + """Build a :class:`box_sdk_gen.BoxClient` from an OAuth2 access token. - ``client_id`` and ``client_secret`` are only required if the caller - wants to let boxsdk refresh the token — most automation callers - already refresh externally, so both default to empty. + ``client_id`` and ``client_secret`` are optional; when given they are passed to the + developer-token auth so the token can be revoked through the SDK. Refreshing the token + stays the caller's job, as most automation callers already refresh it externally. """ if not isinstance(access_token, str) or not access_token: raise BoxException("access_token must be a non-empty string") - boxsdk = _import_boxsdk() - oauth = boxsdk.OAuth2( - client_id=client_id, - client_secret=client_secret, - access_token=access_token, - ) - self.client = boxsdk.Client(oauth) + sdk = import_box_sdk_gen() + config = None + if client_id or client_secret: + config = sdk.DeveloperTokenConfig( + client_id=client_id or None, client_secret=client_secret or None + ) + auth = sdk.BoxDeveloperTokenAuth(token=access_token, config=config) + self.client = sdk.BoxClient(auth=auth) file_automation_logger.info("BoxClient: client ready") return self.client def require_client(self) -> Any: + """Return the initialised SDK client; raise :class:`BoxException` before ``later_init``.""" if self.client is None: raise BoxException("BoxClient not initialised; call later_init() first") return self.client diff --git a/automation_file/remote/box/delete_ops.py b/automation_file/remote/box/delete_ops.py index 09f2259..b3d6350 100644 --- a/automation_file/remote/box/delete_ops.py +++ b/automation_file/remote/box/delete_ops.py @@ -11,7 +11,7 @@ def box_delete_file(file_id: str) -> bool: """Delete a Box file by id.""" client = box_instance.require_client() try: - client.file(file_id=file_id).delete() + client.files.delete_file_by_id(file_id) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_delete_file failed: {error}") from error file_automation_logger.info("box_delete_file: %s", file_id) @@ -22,7 +22,7 @@ def box_delete_folder(folder_id: str, recursive: bool = False) -> bool: """Delete a Box folder by id (optionally recursive).""" client = box_instance.require_client() try: - client.folder(folder_id=folder_id).delete(recursive=recursive) + client.folders.delete_folder_by_id(folder_id, recursive=recursive) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_delete_folder failed: {error}") from error file_automation_logger.info("box_delete_folder: %s (recursive=%s)", folder_id, recursive) diff --git a/automation_file/remote/box/download_ops.py b/automation_file/remote/box/download_ops.py index d055128..313bdb7 100644 --- a/automation_file/remote/box/download_ops.py +++ b/automation_file/remote/box/download_ops.py @@ -16,7 +16,7 @@ def box_download_file(file_id: str, target_path: str) -> bool: target.parent.mkdir(parents=True, exist_ok=True) try: with open(target, "wb") as writer: - client.file(file_id=file_id).download_to(writer) + client.downloads.download_file_to_output_stream(file_id, writer) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_download_file failed: {error}") from error file_automation_logger.info("box_download_file: %s -> %s", file_id, target) diff --git a/automation_file/remote/box/list_ops.py b/automation_file/remote/box/list_ops.py index 58a5b96..f4b12a8 100644 --- a/automation_file/remote/box/list_ops.py +++ b/automation_file/remote/box/list_ops.py @@ -9,6 +9,11 @@ from automation_file.remote.box.client import box_instance +def _type_name(item_type: Any) -> str: + """``"file"``, ``"folder"`` or ``"web_link"``: the SDK reports the type as an enum.""" + return str(getattr(item_type, "value", item_type)) + + def box_list_folder(folder_id: str = "0", limit: int = 100) -> list[dict[str, Any]]: """List entries in a Box folder; return basic metadata per entry. @@ -19,13 +24,12 @@ def box_list_folder(folder_id: str = "0", limit: int = 100) -> list[dict[str, An """ client = box_instance.require_client() try: - folder = client.folder(folder_id=folder_id) - items = folder.get_items(limit=limit) + items = client.folders.get_folder_items(folder_id, limit=limit).entries or [] entries = [ { "id": str(getattr(item, "id", "")), - "name": getattr(item, "name", ""), - "type": getattr(item, "type", "file"), + "name": getattr(item, "name", "") or "", + "type": _type_name(getattr(item, "type", "file")), } for item in items ] diff --git a/automation_file/remote/box/upload_ops.py b/automation_file/remote/box/upload_ops.py index 2e2a0d9..6d37da1 100644 --- a/automation_file/remote/box/upload_ops.py +++ b/automation_file/remote/box/upload_ops.py @@ -7,7 +7,7 @@ from automation_file.exceptions import BoxException, FileNotExistsException from automation_file.logging_config import file_automation_logger from automation_file.remote._upload_tree import walk_and_upload -from automation_file.remote.box.client import box_instance +from automation_file.remote.box.client import box_instance, import_box_sdk_gen def box_upload_file(file_path: str, parent_folder_id: str = "0", name: str = "") -> str: @@ -22,19 +22,20 @@ def box_upload_file(file_path: str, parent_folder_id: str = "0", name: str = "") raise FileNotExistsException(str(local)) client = box_instance.require_client() target_name = name or local.name + sdk = import_box_sdk_gen() + attributes = sdk.UploadFileAttributes( + name=target_name, parent=sdk.UploadFileAttributesParentField(id=parent_folder_id) + ) try: - folder = client.folder(folder_id=parent_folder_id) - new_file = folder.upload(file_path=str(local), file_name=target_name) + with open(local, "rb") as stream: + uploaded = client.uploads.upload_file(attributes, stream) + new_id = str(uploaded.entries[0].id) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_upload_file failed: {error}") from error file_automation_logger.info( - "box_upload_file: %s -> %s/%s (id=%s)", - local, - parent_folder_id, - target_name, - getattr(new_file, "id", "?"), + "box_upload_file: %s -> %s/%s (id=%s)", local, parent_folder_id, target_name, new_id ) - return str(getattr(new_file, "id", "")) + return new_id def box_upload_dir(dir_path: str, parent_folder_id: str = "0") -> list[str]: diff --git a/automation_file/remote/webdav/client.py b/automation_file/remote/webdav/client.py index 08b0e8c..ab787f6 100644 --- a/automation_file/remote/webdav/client.py +++ b/automation_file/remote/webdav/client.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from pathlib import Path from types import TracebackType +from typing import Any from urllib.parse import quote, unquote, urlparse import requests @@ -91,7 +92,7 @@ def _url_for(self, remote_path: str) -> str: return self._base_url + "/" return f"{self._base_url}/{quote(remote_path, safe='/')}" - def _request(self, method: str, remote_path: str, **kwargs: object) -> requests.Response: + def _request(self, method: str, remote_path: str, **kwargs: Any) -> requests.Response: url = self._url_for(remote_path) try: response = self._session.request( diff --git a/dev.toml b/dev.toml index ec2af07..8e80e96 100644 --- a/dev.toml +++ b/dev.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" name = "automation_file_dev" version = "0.0.33" authors = [ - { name = "JE-Chen", email = "zenmailman@gmail.com" }, + { name = "JE-Chen", email = "jechenmailman@gmail.com" }, ] description = "JSON-driven file, Drive, and cloud automation framework (dev channel)." readme = { file = "README.md", content-type = "text/markdown" } @@ -25,15 +25,15 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=47.0.0", - "prometheus_client>=0.25.0", + "cryptography>=50.0.0", + "prometheus_client>=0.26.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=15.0.0", - "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.41.1", - "msal>=1.36.0", - "boxsdk>=3.14.0,<4", + "pyarrow>=25.0.1", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", + "msal>=1.39.0", + "boxsdk>=10.15.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ @@ -60,7 +60,7 @@ dev = [ automation_file_mcp = "automation_file.server.mcp_server:_cli" [project.urls] -"Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" +"Homepage" = "https://github.com/Integration-Automation/FileAutomation" [tool.setuptools.packages] find = { namespaces = false } diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md new file mode 100644 index 0000000..64c6826 --- /dev/null +++ b/docs/updates/2026-09.md @@ -0,0 +1,123 @@ +# 2026-09 update log + +Index and query commands: [README.md](README.md). New entries go at the end. + +--- + +## U-20260922-01 · 2026-09-22 · Adopt progress/architecture/docs-updates rules · #docs #migration + +- **What**: `CLAUDE.md` gained the workspace rule section (commit at every stage; `progress.md` holds open items only; finished work is recorded in `docs/updates/`; `architecture.md` is the short overview). Added `architecture.md` and this `docs/updates/` index. `progress.md` starts with the open items found in the workspace inventory of 2026-09-22 (`D:\Codes\docs\updates\2026-09.md`, U-20260922-01..05). +- **Files**: `CLAUDE.md`, `progress.md`, `architecture.md`, `docs/updates/`. +- **Open items**: see `progress.md`. + +## U-20260922-02 · 2026-09-22 · Stop tracking .idea/ · #done #housekeeping + +- **What**: removed the JetBrains settings under `.idea/` from version control with `git rm -r --cached .idea` (6 files); the files stay on disk. Workspace item W-4 (`D:\Codes\progress.md`). +- **Files**: `.idea/` (untracked); `.gitignore` already ignored `.idea/`. +- **Open items**: none. + +## U-20260922-03 · 2026-09-22 · Delete four merged local branches and fast-forward main · #done #housekeeping + +- **What**: deleted the local branches `codacy-fixes-all-issues` (9b2531f), `feat/new-ops-and-backends` (55d2041), `fix/action-executor-keys` (d1b4f20) and `fix/codacy-bandit-test-secret` (c1a43c2); each is an ancestor of `origin/main` and its PR (#57, #59, #60, #61, #63) was merged. Local `main` was fast-forwarded to `origin/main` (it was 3 behind). Closes `progress.md` #4 (workspace W-5). The remote branches are untouched. +- **Files**: `progress.md`. +- **Open items**: none. + +## U-20260922-04 · 2026-09-22 · Point project URLs at the current repository · #done #metadata + +- **What**: replaced the misspelled or renamed GitHub owner in the project URLs (`Intergration-Automation-Testing/…`, and where present `JE-Chen/je_editor`, `JE-Chen/Integration-testing-environment`, the `AutoControl`/`AutomationEditor`/`AutomationFile`/`JEditor` repo names) with the current `origin`. The `Homepage` in `stable.toml` and `dev.toml` pointed at `JE-Chen/Integration-testing-environment`; the author email was `zenmailman@gmail.com` while every other workspace package uses `jechenmailman@gmail.com`. Workspace item X-11. +- **Files**: `dev.toml`, `stable.toml`. +- **Open items**: none. + +## U-20260922-05 · 2026-09-22 · Contract test for the legacy CLI flags · #done #tests + +- **What**: added `tests/test_legacy_cli_contract.py`. It runs `python -m automation_file` from this checkout as a child process (cwd in `tmp_path`) and checks `-e`/`--execute_file`, `-d`/`--execute_dir`, `--execute_str` encoded exactly as PyBreeze encodes it (a second `json.dumps` on Windows), `-c`/`--create_project`, a non-zero exit with no arguments, and that `create_project_dir` stays importable from the package. The action is `FA_create_file` into `tmp_path`, so a passing run proves it executed. Workspace item X-7. +- **Checks**: the file passes on its own three times in a row (about 80 s, mostly nine interpreter start-ups); full suite 768 passed. One earlier full run failed a single case while AutoControlGUI's editable tree (loaded as a pytest plugin) was mid-edit in another session; it did not recur. ruff clean. +- **Files**: `tests/test_legacy_cli_contract.py`, `architecture.md` §6, `progress.md`. +- **Open items**: none. + +## U-20260923-01 · 2026-09-23 · Dependency floors raised; Dependabot on dev; boxsdk 10 refused · #done #deps #ci + +- **What** (`progress.md` #3, workspace X-16): the five Dependabot PRs sat unmerged since June because they target `main`, the released branch. Four of them are applied here on `dev` instead, in both `dev.toml` and `stable.toml`: `cryptography>=49.0.0` (#87), `pyarrow>=24.0.0` (#89), `opentelemetry-sdk>=1.42.1` (#88), `msal>=1.37.0` (#83). Those PRs are closed with a pointer to this commit. +- **boxsdk stays at `>=3.14.0,<4`** and PR #86 (`>=10.12.0,<11`) is closed as breaking: the `boxsdk` distribution's 10.x line installs the module **`box_sdk_gen`**, not `boxsdk` (verified by installing `boxsdk==10.15.0` into a scratch venv: `site-packages/box_sdk_gen` only, `import boxsdk` raises `ModuleNotFoundError`). `automation_file/remote/box/client.py` imports `boxsdk` and builds `boxsdk.OAuth2` / `boxsdk.Client`, so the cap is what keeps the Box backend working. Porting it to the new SDK is now `progress.md` #6. +- **Dependabot**: `.github/dependabot.yml` sets `target-branch: "dev"`, so future updates land where work happens and ship with the next release. +- **Checks**: `pytest tests` 752 passed, 8 skipped (metadata-only change; the optional extras are not installed locally). +- **Files**: `dev.toml`, `stable.toml`, `.github/dependabot.yml`, `progress.md`. +- **Open items**: `progress.md` #6 (Box SDK port). `origin/release/bump-v0.0.32` is still unmerged, which is the rest of #3. + +## U-20260923-02 · 2026-09-23 · CLAUDE.md matches the code again · #done #docs + +- **What** (`progress.md` #1, #2 and the documentation half of #5): + - `CLAUDE.md`'s package tree listed a 2026-04 layout: no `ftp/`, `onedrive/`, `box/`, `smb/`, `webdav/`, no MCP server, web UI or metrics server, no `client/`, `trigger/`, `scheduler/`, `notify/`, and a nine-tab GUI that has since become home / local / HTTP / JSON editor / servers / scheduler / trigger / progress with the cloud backends grouped under the transfer tab. It is replaced by a compact map that matches `architecture.md` §2, with a line asking to keep the two in step. + - CI is described as it runs: a `lint` job on Ubuntu (Python 3.12), then `pytest` on Windows across 3.10–3.14 (it said 3.10–3.12). + - The HTTP server section said only `POST /actions` is handled; it also serves `GET /healthz`, `/readyz`, `/openapi.json` and `/progress`, which only report. + - The Strategy bullet now names every `remote//*_ops.py`, not just Google Drive. + - `.codacy_tmp/` was already ignored by `ce13593` (#1); the item just had not been removed. +- **Files**: `CLAUDE.md`, `progress.md`. +- **Open items**: `progress.md` #5 keeps only the port clash (9944 is also MailThunder's socket-server default). + +## U-20260923-03 · 2026-09-23 · FileAutomation.log moves out of the working directory · #done #logging + +- **What** (workspace item X-6): `logging_config.py` opened `FileAutomation.log` in the working directory at import, so every process that imported the package — PyBreeze, TestPioneer, test runs — left one wherever it started (`D:\Codes`, JEditor's and WebRunner's checkouts had collected them). It now follows the workspace scheme: `$FILE_AUTOMATION_LOG_FILE` or `~/.automation_file/logs/FileAutomation.log`, opened on the first record, append mode with the process id on each line, rotation to `.1` past 10 MiB when a process opens it, UTF-8 with `backslashreplace`, and `os.devnull` plus one warning when the path cannot be opened. +- **Also**: the "built default registry with N commands" message runs at import and was INFO, which the stderr handler mirrors, so every import — `python -m automation_file --help` included — printed it. It is DEBUG now (still in the file, no longer on stderr). +- **Tests**: `tests/test_log_location.py` (5): home default, environment override, loading the module alone leaves the cwd empty, the first record creates the directory and later handlers append (CJK, non-cp950, lone surrogate), an unopenable path only warns. Suite: 757 passed, 8 skipped; ruff check and format clean, mypy clean on the module. +- **Files**: `automation_file/logging_config.py`, `automation_file/core/action_registry.py`, `tests/test_log_location.py`, `architecture.md` §2. +- **Open items**: none here. + +## U-20260923-04 · 2026-09-23 · Port clash with MailThunder resolved on MailThunder's side · #done #docs + +- **What** (`progress.md` #5): MailThunder's socket-server default moved from 9944 to 9942 (MailThunder U-20260923-06). This package's defaults are unchanged: TCP 9943, HTTP 9944, metrics 9945. Keeping them avoided changing the port in every README, doc page, CLI default and GUI spin box here. +- **Files**: `architecture.md` (§6 records the new MailThunder default), `progress.md`. +- **Open items**: none. + +## U-20260923-05 · 2026-09-23 · Box backend moves to box_sdk_gen (boxsdk 10) · #done #deps #box + +- **What** (`progress.md` #6): the Box backend used the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), which kept the dependency capped at `<4`. Box consolidated its SDKs: from 10.0 the `boxsdk` distribution ships only `box_sdk_gen`, and Box recommends migrating to 10.x, with support for the consolidated SDKs continuing into 2027. The backend now uses `box_sdk_gen`: + - `BoxClient.later_init` builds `BoxDeveloperTokenAuth` (with `DeveloperTokenConfig` when a client id or secret is given) and `box_sdk_gen.BoxClient`. The signature is unchanged. + - upload → `client.uploads.upload_file(UploadFileAttributes(name, parent=…), stream)`, returning `entries[0].id` + - download → `client.downloads.download_file_to_output_stream` + - list → `client.folders.get_folder_items(folder_id, limit=…)`. The item type is an enum there, so `box_list_folder` still returns `"file"`, `"folder"` or `"web_link"` strings. + - delete → `client.files.delete_file_by_id` and `client.folders.delete_folder_by_id(…, recursive=…)` +- **Dependency**: `boxsdk>=10.0.0,<11` in `requirements.txt`, `dev.toml` and `stable.toml`. The 3.x cap is gone. +- **How it was checked**: boxsdk 10.15.0 was installed in a scratch venv and each signature used above was read from the installed package. The FileAutomation venv now has boxsdk 10.7.0. +- **Tests**: + - `tests/test_box_ops.py`: the fakes now mirror the SDK's managers (`uploads`, `downloads`, `folders`, `files`), and the tests check the arguments each call receives (name, parent id, content, recursive flag, the ids deleted). + - New: `later_init` builds a real `box_sdk_gen.BoxClient`. + - Full suite: 759 passed, 8 skipped. + - There are still no calls against a live Box account. +- **Files**: `automation_file/remote/box/{client,upload_ops,download_ops,list_ops,delete_ops}.py`, `tests/test_box_ops.py`, `requirements.txt`, `dev.toml`, `stable.toml`, `progress.md`. +- **Open items**: none. + +## U-20260923-06 · 2026-09-23 · cryptography floor 50 and msal 1.39 · #done #deps #security + +- **Found by**: running `pip-audit` against the local venv. The venv had cryptography 46.0.7, and advisories PYSEC-2026-3552 and PYSEC-2026-3554 are fixed in 49.0.0 and 50.0.0. The declared floor `cryptography>=49.0.0` still allowed a 49.x without the 50.0.0 fix. +- **Change**: + - `cryptography>=50.0.0` in `dev.toml` and `stable.toml`. + - `msal>=1.39.0` there and in `requirements.txt`. msal releases before 1.39 require `cryptography<49`, so the two floors have to move together. +- **Local venv**: the flagged packages were upgraded (cryptography 50.0.1, msal 1.39.0, httplib2, idna, pyasn1, urllib3, pip), and `uv pip check` is clean. Suite: 759 passed, 8 skipped. +- **Files**: `dev.toml`, `stable.toml`, `requirements.txt`. +- **Open items**: none. + +## U-20260923-07 · 2026-09-23 · Stale release/bump-v0.0.32 branch deleted · #done #housekeeping + +- **What** (`progress.md` #3, workspace X-16): `origin/release/bump-v0.0.32` held a single bot commit, `0014231` ("Bump version to v0.0.32 [skip ci]", 2026-04-21), which changed only the version lines in `dev.toml` and `stable.toml`. `main` is at 0.0.47, so merging it would move the version backwards. It had no pull request. The branch was deleted from `origin`. If it is ever needed, restore it with `git push origin 0014231:refs/heads/release/bump-v0.0.32`. +- **Open items**: none. + +## U-20260923-08 · 2026-09-23 · dev CI green again: format check and mypy · #done #ci + +- **What**: every push to `dev` since 2026-09-22 failed the `lint` job at `ruff format --check`, eight runs in a row. The files were `automation_file/remote/box/client.py` and `tests/test_box_ops.py` (U-20260923-05) and `tests/test_legacy_cli_contract.py` (an earlier change). The pytest jobs never ran behind it. All three are formatted. +- **Also**: `mypy automation_file` reported 11 errors in `remote/webdav/client.py`, because `_request(**kwargs: object)` cannot satisfy the requests type stubs. CI does not install those stubs, so it would not have seen the errors. The kwargs are now typed `Any`, and mypy passes with the stubs as well. +- **Result**: run for `1874801`: lint and pytest on 3.10–3.14 all pass. +- **Open items**: none. + +## U-20260923-09 · 2026-09-23 · Five floors from the Dependabot PRs on main; the PRs closed · #done #deps + +- **What**: five Dependabot PRs still targeted `main` (#91–#95), from before Dependabot was pointed at `dev`. Their floors are applied on `dev`, the same way this morning's four were (U-20260923-01): + - `prometheus_client>=0.26.0` + - `opentelemetry-api>=1.44.0` and `opentelemetry-sdk>=1.44.0` + - `pyarrow>=25.0.1` + - `boxsdk>=10.15.0,<11` + + They are in `dev.toml`, `stable.toml` and `requirements.txt`. `requirements.txt` still had `pyarrow>=15.0.0`, which is now in step with the TOMLs. +- **Checked**: the local venv was upgraded to those versions, `uv pip check` is clean, and the suite gives 759 passed, 8 skipped. +- **PRs**: #91–#95 are closed with a comment pointing at the `dev` commit. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md new file mode 100644 index 0000000..5fd76ff --- /dev/null +++ b/docs/updates/README.md @@ -0,0 +1,80 @@ +# docs/updates: update log index + +`progress.md` holds only work that is **not done yet**. Everything that *was* done (what changed, measured numbers, decisions, snapshots) is recorded here: **one batch file per month**, one entry per piece of work, each entry with a fixed-format ID and tags, and one row per entry in the index below. + +> No TODOs here. If an entry mentions something still open, it only points to it (e.g. "open item: `progress.md` #3"); the item itself lives in `progress.md`. + +## How to query + +Run from the repository root: + +| To find | Command | +|---|---| +| every entry, one line each | `rg -n "^## U-2" docs/updates` | +| entries of one type | `rg -n "^## U-2.*#done" docs/updates` | +| entries with a topic tag | `rg -n "^## U-2.*#" docs/updates` | +| one day or one month | `rg -n "^## U-202609" docs/updates` | +| the full text of one entry | `rg -n -A 60 "^## U-20260922-01" docs/updates` | +| any keyword | `rg -n "keyword" docs/updates` | + +Without `rg`: `git grep -n "^## U-2" -- docs/updates`, or in PowerShell `Select-String -Path docs/updates/*.md -Pattern '^## U-2'`. + +## Entry format + +```markdown +## U-YYYYMMDD-NN · YYYY-MM-DD · one-line title · #type #topic + +- **What**: ... +- **Result / numbers**: ... +- **Files**: `path` ... +- **Evidence**: commit, file:line, link ... +- **Open items**: none / see `progress.md` ... +``` + +- **ID**: `U-` + date + two-digit sequence for that day. IDs are never renumbered or reused, so code comments and other documents can cite them. +- **Type tag** (exactly one): `#done` finished `progress.md` item, `#snapshot` measurement or inventory, `#decision`, `#incident`, `#migration`, `#docs`, `#release`. +- Topic tags are free-form (`#mcp`, `#wayland`, ...). +- Keep conclusions, numbers, files and evidence; drop the reasoning trail and dead ends. + +## Batch rules + +1. One file per month: `docs/updates/YYYY-MM.md`. Append new entries at the end. +2. Over about 800 lines, continue in `YYYY-MM-b.md` (then `-c`) and list it in the batch table below. +3. **Claim the ID under a lock.** Several sessions may write this log at the same time (for example parallel autonomous runs), and without a lock two of them pick the same number: + 1. `mkdir docs/updates/.id-lock`. Creating a directory is atomic, so only one writer succeeds. If it already exists, someone else is claiming: wait a few seconds and retry. A lock older than 10 minutes is stale and may be removed. + 2. Find the day's last number with `rg -n "^## U-YYYYMMDD" docs/updates` and write the heading line and the index row. + 3. `rmdir docs/updates/.id-lock`, then fill in the body. Git never tracks the empty lock directory. + 4. Before committing, `rg -c "^## U-" docs/updates` must report one match in total. If not, renumber your entry under the lock and fix its index row. Whoever merges a branch renumbers entries that reuse an ID. +4. **One line per index row**: title only (about 60 characters), no summary. +5. Never rewrite a recorded entry. Correct it with a new `#decision` or `#incident` entry and add "→ corrected in U-..." to the old one. + +## When a `progress.md` item is done + +In the same commit: delete the item from `progress.md`, add a `#done` entry here that names it, and add its index row. + +--- + +## Index (newest first) + +| ID | Date | Title | Tags | Batch | +|---|---|---|---|---| +| U-20260923-09 | 2026-09-23 | Five floors from the Dependabot PRs on main; the PRs closed | #done #deps | [2026-09](2026-09.md) | +| U-20260923-08 | 2026-09-23 | dev CI green again: format check and mypy | #done #ci | [2026-09](2026-09.md) | +| U-20260923-07 | 2026-09-23 | Stale release/bump-v0.0.32 branch deleted | #done #housekeeping | [2026-09](2026-09.md) | +| U-20260923-06 | 2026-09-23 | cryptography floor 50 and msal 1.39 | #done #deps #security | [2026-09](2026-09.md) | +| U-20260923-05 | 2026-09-23 | Box backend moves to box_sdk_gen (boxsdk 10) | #done #deps #box | [2026-09](2026-09.md) | +| U-20260923-04 | 2026-09-23 | Port clash with MailThunder resolved on MailThunder's side | #done #docs | [2026-09](2026-09.md) | +| U-20260923-03 | 2026-09-23 | FileAutomation.log moves out of the working directory | #done #logging | [2026-09](2026-09.md) | +| U-20260923-02 | 2026-09-23 | CLAUDE.md matches the code again | #done #docs | [2026-09](2026-09.md) | +| U-20260923-01 | 2026-09-23 | Dependency floors raised; Dependabot on dev; boxsdk 10 refused | #done #deps #ci | [2026-09](2026-09.md) | +| U-20260922-05 | 2026-09-22 | Contract test for the legacy CLI flags | #done #tests | [2026-09](2026-09.md) | +| U-20260922-04 | 2026-09-22 | Point project URLs at the current repository | #done #metadata | [2026-09](2026-09.md) | +| U-20260922-03 | 2026-09-22 | Delete four merged local branches and fast-forward main | #done #housekeeping | [2026-09](2026-09.md) | +| U-20260922-02 | 2026-09-22 | Stop tracking .idea/ | #done #housekeeping | [2026-09](2026-09.md) | +| U-20260922-01 | 2026-09-22 | Adopt progress/architecture/docs-updates rules | #docs #migration | [2026-09](2026-09.md) | + +## Batches + +| File | Period | Entries | +|---|---|---:| +| [2026-09.md](2026-09.md) | 2026-09 | 14 | diff --git a/progress.md b/progress.md new file mode 100644 index 0000000..77dfd04 --- /dev/null +++ b/progress.md @@ -0,0 +1,8 @@ +# progress.md: FileAutomation + +Outstanding work only. When an item is done, delete it in the same commit and add a `#done` entry to `docs/updates/` (format and query commands: `docs/updates/README.md`). No finished items, no history, no rules (rules live in `CLAUDE.md`). +Item numbers (`#n`) are never reused. Tags: [DECIDE] needs the owner's decision, [BLOCKED] waits on something else, [UNVERIFIED] observed but not confirmed. +Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-16). + +## Open + diff --git a/requirements.txt b/requirements.txt index 76e9db9..c4c1b88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,9 @@ tqdm watchdog defusedxml>=0.7.1 PyYAML>=6.0.3 -pyarrow>=15.0.0 -opentelemetry-api>=1.41.1 -opentelemetry-sdk>=1.41.1 -msal>=1.36.0 -boxsdk>=3.14.0,<4 +pyarrow>=25.0.1 +opentelemetry-api>=1.44.0 +opentelemetry-sdk>=1.44.0 +msal>=1.39.0 +boxsdk>=10.15.0,<11 tomli; python_version<"3.11" \ No newline at end of file diff --git a/stable.toml b/stable.toml index e60cad9..02f55a0 100644 --- a/stable.toml +++ b/stable.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" name = "automation_file" version = "0.0.31" authors = [ - { name = "JE-Chen", email = "zenmailman@gmail.com" }, + { name = "JE-Chen", email = "jechenmailman@gmail.com" }, ] description = "JSON-driven file, Drive, and cloud automation framework." readme = { file = "README.md", content-type = "text/markdown" } @@ -25,15 +25,15 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=47.0.0", - "prometheus_client>=0.25.0", + "cryptography>=50.0.0", + "prometheus_client>=0.26.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=15.0.0", - "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.41.1", - "msal>=1.36.0", - "boxsdk>=3.14.0,<4", + "pyarrow>=25.0.1", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", + "msal>=1.39.0", + "boxsdk>=10.15.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ @@ -60,7 +60,7 @@ dev = [ automation_file_mcp = "automation_file.server.mcp_server:_cli" [project.urls] -"Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" +"Homepage" = "https://github.com/Integration-Automation/FileAutomation" [tool.setuptools.packages] find = { namespaces = false } diff --git a/tests/test_box_ops.py b/tests/test_box_ops.py index ab88d07..10337fb 100644 --- a/tests/test_box_ops.py +++ b/tests/test_box_ops.py @@ -2,15 +2,17 @@ Live Box endpoints are outside CI; these tests verify registry wiring, the Client singleton's guard clauses, and the error-path wrapping that -converts ``boxsdk`` failures into :class:`BoxException`. +converts ``box_sdk_gen`` failures into :class:`BoxException`. """ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest +from box_sdk_gen.schemas.file_base import FileBaseTypeField from automation_file import ( BoxClient, @@ -25,52 +27,59 @@ class _FakeItem: - def __init__(self, item_id: str, name: str, item_type: str = "file") -> None: + def __init__(self, item_id: str, name: str, item_type: Any) -> None: self.id = item_id self.name = name self.type = item_type -class _FakeFile: - def __init__(self, file_id: str) -> None: - self.id = file_id +class _Uploads: + def __init__(self) -> None: + self.calls: list[tuple[str, str, bytes]] = [] - def download_to(self, writer: Any) -> None: - writer.write(b"contents") + def upload_file(self, attributes: Any, file: Any) -> Any: + self.calls.append((attributes.name, attributes.parent.id, file.read())) + return SimpleNamespace(entries=[SimpleNamespace(id="new-id")]) - def delete(self) -> None: - return None +class _Downloads: + def download_file_to_output_stream(self, file_id: str, output_stream: Any) -> None: + output_stream.write(f"contents of {file_id}".encode()) -class _FakeFolder: - def __init__(self, folder_id: str) -> None: - self.id = folder_id - self._uploads: list[tuple[str, str]] = [] - def upload(self, file_path: str, file_name: str) -> _FakeFile: - self._uploads.append((file_path, file_name)) - return _FakeFile("new-id") +class _Folders: + def __init__(self) -> None: + self.deleted: list[tuple[str, bool]] = [] - def get_items(self, limit: int = 100) -> list[_FakeItem]: - del limit - return [_FakeItem("1", "a.txt"), _FakeItem("2", "subdir", "folder")] + def get_folder_items(self, folder_id: str, *, limit: int | None = None) -> Any: + del folder_id, limit + return SimpleNamespace( + entries=[ + _FakeItem("1", "a.txt", FileBaseTypeField.FILE), + _FakeItem("2", "subdir", "folder"), + ] + ) - def delete(self, recursive: bool = False) -> None: - del recursive + def delete_folder_by_id(self, folder_id: str, *, recursive: bool | None = None) -> None: + self.deleted.append((folder_id, bool(recursive))) -class _FakeBoxClient: +class _Files: def __init__(self) -> None: - self._files: dict[str, _FakeFile] = {} - self._folders: dict[str, _FakeFolder] = {} + self.deleted: list[str] = [] - def file(self, file_id: str) -> _FakeFile: - self._files.setdefault(file_id, _FakeFile(file_id)) - return self._files[file_id] + def delete_file_by_id(self, file_id: str) -> None: + self.deleted.append(file_id) - def folder(self, folder_id: str) -> _FakeFolder: - self._folders.setdefault(folder_id, _FakeFolder(folder_id)) - return self._folders[folder_id] + +class _FakeBoxClient: + """Mirrors the managers of :class:`box_sdk_gen.BoxClient` that the backend calls.""" + + def __init__(self) -> None: + self.uploads = _Uploads() + self.downloads = _Downloads() + self.folders = _Folders() + self.files = _Files() @pytest.fixture(name="fake_box") @@ -129,16 +138,17 @@ def test_upload_dir_uploads_each_file(tmp_path: Path, fake_box: _FakeBoxClient) (tmp_path / "sub" / "b.txt").write_text("b", encoding="utf-8") uploaded_keys = upload_ops.box_upload_dir(str(tmp_path)) assert sorted(uploaded_keys) == ["a.txt", "sub/b.txt"] - folder = fake_box.folder("0") - flat_names = sorted(name for _, name in folder._uploads) # pylint: disable=protected-access - assert flat_names == ["a.txt", "sub/b.txt"] + assert sorted((name, parent) for name, parent, _ in fake_box.uploads.calls) == [ + ("a.txt", "0"), + ("sub/b.txt", "0"), + ] def test_download_writes_target(tmp_path: Path, fake_box: _FakeBoxClient) -> None: del fake_box target = tmp_path / "out" / "f.txt" assert download_ops.box_download_file("42", str(target)) is True - assert target.read_bytes() == b"contents" + assert target.read_bytes() == b"contents of 42" def test_list_folder_returns_entries(fake_box: _FakeBoxClient) -> None: @@ -151,13 +161,13 @@ def test_list_folder_returns_entries(fake_box: _FakeBoxClient) -> None: def test_delete_file_uses_client(fake_box: _FakeBoxClient) -> None: - del fake_box assert delete_ops.box_delete_file("7") is True + assert fake_box.files.deleted == ["7"] def test_delete_folder_uses_client(fake_box: _FakeBoxClient) -> None: - del fake_box assert delete_ops.box_delete_folder("7", recursive=True) is True + assert fake_box.folders.deleted == [("7", True)] def test_errors_in_sdk_surface_as_box_exception( @@ -166,6 +176,25 @@ def test_errors_in_sdk_surface_as_box_exception( def blow(*_a: Any, **_k: Any) -> None: raise RuntimeError("simulated SDK error") - monkeypatch.setattr(fake_box, "folder", blow) + monkeypatch.setattr(fake_box.folders, "get_folder_items", blow) with pytest.raises(BoxException): list_ops.box_list_folder() + + +def test_upload_file_sends_name_parent_and_content( + tmp_path: Path, fake_box: _FakeBoxClient +) -> None: + src = tmp_path / "report.txt" + src.write_bytes(b"payload") + upload_ops.box_upload_file(str(src), parent_folder_id="99", name="renamed.txt") + assert fake_box.uploads.calls == [("renamed.txt", "99", b"payload")] + + +def test_later_init_builds_a_real_sdk_client() -> None: + import box_sdk_gen + + client = BoxClient() + built = client.later_init("token-value", client_id="id", client_secret="secret") + assert isinstance(built, box_sdk_gen.BoxClient) + assert client.require_client() is built + assert hasattr(built, "uploads") and hasattr(built, "folders") diff --git a/tests/test_legacy_cli_contract.py b/tests/test_legacy_cli_contract.py new file mode 100644 index 0000000..c1285f7 --- /dev/null +++ b/tests/test_legacy_cli_contract.py @@ -0,0 +1,98 @@ +"""Contract test for the legacy CLI flags that other repositories call. + +PyBreeze starts ``python -m automation_file --execute_str `` (JSON-encoded a second +time on Windows, see ``pybreeze/extend/process_executor/python_task_process_manager.py``) +and ``--execute_file ``; TestPioneer's ``parallel_run`` starts +``--execute_file ``; PyBreeze's "create project" menu calls +``automation_file.create_project_dir()`` in process. Nothing on the consumer side tests +these, so this file does: renaming or removing a flag, or dropping the second +JSON decode on Windows, breaks them (workspace item X-7). +""" + +import json +import os +import subprocess # nosec B404 - the CLI is exercised as a real child process +import sys +from pathlib import Path + +import pytest + +import automation_file + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACKAGE = "automation_file" +IS_WINDOWS = sys.platform in ("win32", "cygwin", "msys") + + +def _run_cli(cwd: Path, *args: str) -> subprocess.CompletedProcess: + """Run ``python -m PACKAGE`` from this checkout; *cwd* catches any log file it writes.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(REPO_ROOT), env.get("PYTHONPATH")])) + env["PYTHONIOENCODING"] = "utf-8" + return subprocess.run( # nosemgrep # nosec B603 - fixed interpreter, test arguments + [sys.executable, "-m", PACKAGE, *args], + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + timeout=300, + check=False, + ) + + +def _pybreeze_execute_str(actions: list) -> str: + """Encode *actions* the way PyBreeze's ``start_test_process`` does.""" + payload = json.dumps(actions) + return json.dumps(payload) if IS_WINDOWS else payload + + +def _actions(target: Path) -> list: + """A harmless action list whose only effect is writing *target*.""" + return [["FA_create_file", {"file_path": str(target), "content": "x"}]] + + +def _assert_ran(result: subprocess.CompletedProcess, target: Path) -> None: + assert result.returncode == 0, result.stderr + assert target.exists(), result.stdout + result.stderr + + +@pytest.mark.parametrize("flag", ["-e", "--execute_file"]) +def test_execute_file(tmp_path, flag): + target = tmp_path / "created.txt" + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps(_actions(target)), encoding="utf-8") + _assert_ran(_run_cli(tmp_path, flag, str(action_file)), target) + + +@pytest.mark.parametrize("flag", ["-d", "--execute_dir"]) +def test_execute_dir(tmp_path, flag): + target = tmp_path / "created.txt" + action_dir = tmp_path / "actions" + action_dir.mkdir() + (action_dir / "actions.json").write_text(json.dumps(_actions(target)), encoding="utf-8") + _assert_ran(_run_cli(tmp_path, flag, str(action_dir)), target) + + +def test_execute_str_as_pybreeze_sends_it(tmp_path): + target = tmp_path / "created.txt" + _assert_ran( + _run_cli(tmp_path, "--execute_str", _pybreeze_execute_str(_actions(target))), target + ) + + +@pytest.mark.parametrize("flag", ["-c", "--create_project"]) +def test_create_project(tmp_path, flag): + project = tmp_path / "project" + result = _run_cli(tmp_path, flag, str(project)) + assert result.returncode == 0, result.stderr + assert project.is_dir(), result.stdout + result.stderr + assert any(project.rglob("*.json")), result.stdout + result.stderr + + +def test_no_flag_exits_non_zero(tmp_path): + assert _run_cli(tmp_path).returncode != 0 + + +def test_create_project_dir_is_exported(): + assert callable(automation_file.create_project_dir) diff --git a/tests/test_log_location.py b/tests/test_log_location.py new file mode 100644 index 0000000..f86be9f --- /dev/null +++ b/tests/test_log_location.py @@ -0,0 +1,80 @@ +"""Where FileAutomation's log goes, and that importing the package writes nothing (X-6).""" + +import logging +import os +import subprocess # nosec B404 - the import is exercised in a fresh interpreter +import sys +import warnings +from pathlib import Path + +from automation_file import logging_config as loggin_instance +from automation_file.logging_config import LOG_FILE_ENV, FileAutomationFileHandler, default_log_file + +MODULE_FILE = Path(loggin_instance.__file__) + +# Load just this module in a fresh interpreter (skipping the package __init__), then list +# the working directory. +_IMPORT_ONLY = ( + "import importlib.util, os, sys\n" + "spec = importlib.util.spec_from_file_location('probe', sys.argv[1])\n" + "spec.loader.exec_module(importlib.util.module_from_spec(spec))\n" + "print(sorted(os.listdir('.')))\n" +) + + +def _log(handler: logging.Handler, message: str) -> None: + log = logging.getLogger(f"test_log_location.{id(handler)}") + log.propagate = False + log.setLevel(logging.DEBUG) + log.addHandler(handler) + try: + log.warning(message) + finally: + log.removeHandler(handler) + handler.close() + + +def test_default_is_under_the_home_directory(monkeypatch, tmp_path): + monkeypatch.delenv(LOG_FILE_ENV, raising=False) + monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path)) + assert default_log_file() == tmp_path / ".automation_file" / "logs" / "FileAutomation.log" + + +def test_environment_variable_overrides_the_location(monkeypatch, tmp_path): + monkeypatch.setenv(LOG_FILE_ENV, str(tmp_path / "custom.log")) + assert default_log_file() == tmp_path / "custom.log" + + +def test_importing_writes_no_file(tmp_path): + target = tmp_path / "home" / "FileAutomation.log" + env = {key: value for key, value in os.environ.items() if key != LOG_FILE_ENV} + env[LOG_FILE_ENV] = str(target) + result = subprocess.run( # nosemgrep # nosec B603 - fixed interpreter, test arguments + [sys.executable, "-c", _IMPORT_ONLY, str(MODULE_FILE)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "[]" + assert not target.parent.exists() + + +def test_first_record_creates_the_directory_and_later_handlers_append(tmp_path): + target = tmp_path / "nested" / "FileAutomation.log" + _log(FileAutomationFileHandler(str(target)), "first") + _log(FileAutomationFileHandler(str(target)), "second 中文 ⠐ \udcff") + text = target.read_text(encoding="utf-8") + assert "first" in text and "second 中文 ⠐" in text and "\\udcff" in text + + +def test_an_unopenable_path_only_warns(tmp_path): + blocker = tmp_path / "a_file" + blocker.write_text("x", encoding="utf-8") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _log(FileAutomationFileHandler(str(blocker / "FileAutomation.log")), "goes nowhere") + assert any(issubclass(item.category, RuntimeWarning) for item in caught)