From 3add1b17b9e7adbf6d8fc93f2db32d0d6f3e82d4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 15:01:05 +0800 Subject: [PATCH 01/20] Add architecture.md, progress.md and docs/updates; commit per stage Work sat uncommitted for weeks in several repositories of this workspace and progress files grew into change logs. The project guidelines now ask for a commit at every stage, keep progress.md to open items only, record finished work in monthly batches under docs/updates/ with an index and query commands, and add a short architecture.md overview. --- CLAUDE.md | 12 +++ architecture.md | 158 ++++++++++++++++++++++++++++++++++++++++ docs/updates/2026-09.md | 11 +++ docs/updates/README.md | 63 ++++++++++++++++ progress.md | 13 ++++ 5 files changed, 257 insertions(+) create mode 100644 architecture.md create mode 100644 docs/updates/2026-09.md create mode 100644 docs/updates/README.md create mode 100644 progress.md diff --git a/CLAUDE.md b/CLAUDE.md index 8fd3887..08006e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -267,6 +267,18 @@ All code must satisfy common static-analysis rules. Review every change against - Before committing any non-trivial change, run `ruff check automation_file/ tests/` locally. - When adding a `# noqa: RULE`, justify it in the comment — never blanket-disable. +## Stage commits, `progress.md`, `docs/updates/` and `architecture.md` + +Workspace rule shared by every repository under `D:\Codes` (full text: `D:\Codes\CLAUDE.md`). + +- **Commit at every stage.** A stage is the smallest piece of work that leaves the repository consistent and passes this project's checks (definition of done, tests, lint): one finished `progress.md` item, or one self-contained step of a larger one. Commit it before starting the next stage, before switching to another repository, and before the session ends. Do not leave work uncommitted across sessions; if a stage cannot be finished, commit the consistent part and record the rest in `progress.md`. + - Stage only the files that stage touched (`git add `, never `git add -A`), follow this file's commit-message rules, and never add AI attribution. + - Committing is not pushing: push or open a PR only as this project's branch flow says or when asked. +- **`progress.md`** (repository root, tracked) holds outstanding work only: no finished items, no history, no rules. +- **`docs/updates/`** records finished work: one batch file per month (`YYYY-MM.md`), one entry per piece of work headed `## U-YYYYMMDD-NN · date · title · #tags`, and an index with query commands in `docs/updates/README.md`. When a `progress.md` item is done, delete it and add a `#done` entry plus its index row in the same commit. +- **`architecture.md`** (repository root) is the short architecture overview: layers, entry points, main flows, extension points, cross-project boundaries. Update it in the same commit whenever a change alters any of those. +- **Cross-project contracts** are listed in `architecture.md` §6: what other repositories rely on here (CLI flags, import paths, constructor arguments, file layouts) and what this repository relies on elsewhere. No test here protects them, so never rename or remove one without changing its consumers in the same round, and update §6 whenever a contract is added or changes. + ## Commit & PR rules - Commit messages: short imperative sentence (e.g., "Fix rename_file overwrite bug", "Update stable version"). diff --git a/architecture.md b/architecture.md new file mode 100644 index 0000000..f08b4d5 --- /dev/null +++ b/architecture.md @@ -0,0 +1,158 @@ +# FileAutomation Architecture + +> Short overview for people and agents. +> Last verified: 2026-09-22 against `2e6c0ec` on `dev`. + +## 1. Purpose + +FileAutomation (`automation_file`) is an automation-first library for local file, directory and +archive operations, SSRF-checked HTTP downloads, and remote storage (Google Drive, S3, Azure Blob, +Dropbox, SFTP, FTP, OneDrive, Box, plus SMB and WebDAV clients). Every operation is an `FA_*` +command in one `ActionRegistry`, so JSON action lists run the same way in-process, from files, from +the CLI, over loopback TCP or HTTP servers, as MCP tools, or from the PySide6 GUI. + +## 2. Layers and directories + +| Path | Responsibility | +| --- | --- | +| `automation_file/__init__.py` | Public facade (`__all__`). Wires the shared `executor`, `callback_executor` and `package_manager` over one registry. `launch_ui` is loaded lazily through `__getattr__` | +| `automation_file/__main__.py` | CLI: legacy flags plus subcommands | +| `automation_file/core/` | Engine: `action_registry.py` (`ActionRegistry`, `build_default_registry`), `action_executor.py` (`ActionExecutor`, shared `executor`), `callback_executor.py`, `package_loader.py`, `plugins.py`, `dag_executor.py`, `action_queue.py`, `json_store.py`, `substitution.py`. Also cross-cutting helpers: `retry`, `quota`, `rate_limit`, `circuit_breaker`, `file_lock`, `sqlite_lock`, `checksum`, `manifest`, `crypto`, `secrets`, `config`, `config_watcher`, `audit`, `metrics`, `tracing`, `progress`, `fim`, `content_store` | +| `automation_file/local/` | Local strategy modules: file, dir, zip, tar and archive ops, sync, diff, text/JSON/data edits, templates, versioning, trash, `shell_ops` (argv-only subprocess), conditional branches. `safe_paths.py` guards against path traversal | +| `automation_file/remote/` | `url_validator.py` (SSRF guard), `http_download.py`, `cross_backend.py`, `fsspec_bridge.py`. One subpackage per backend: `google_drive/`, `s3/`, `azure_blob/`, `dropbox_api/`, `sftp/`, `ftp/`, `onedrive/`, `box/`, each with `client.py`, `*_ops.py` and `register__ops`. `smb/` and `webdav/` have a client only | +| `automation_file/server/` | `tcp_server.py`, `http_server.py`, `mcp_server.py`, `web_ui.py`, `metrics_server.py`, `action_acl.py` (`ActionACL`), `network_guards.py` (`ensure_loopback`) | +| `automation_file/client/` | `HTTPActionClient` for the HTTP action server | +| `automation_file/trigger/`, `scheduler/`, `notify/` | Watchdog file triggers, cron scheduler, notification sinks. Each registers its own `FA_*` ops | +| `automation_file/project/` | `ProjectBuilder`, `create_project_dir` | +| `automation_file/ui/` | PySide6 GUI: `launcher.launch_ui`, `main_window.MainWindow`, `worker.ActionWorker`, `log_widget.LogPanel`, `tabs/` (backend panels are grouped under `TransferTab`) | +| `automation_file/utils/` | File discovery, fast find, grep, duplicate finder, backup rotation | +| `automation_file/exceptions.py`, `logging_config.py` | `FileAutomationException` hierarchy; `file_automation_logger` | +| `stable.toml`, `dev.toml` | Packaging for `automation_file` and `automation_file_dev`. No `pyproject.toml` is committed; CI and publish copy one of these TOMLs into place | +| `main_ui.py` | Development shortcut for `launch_ui()` | +| `tests/`, `docs/`, `examples/mcp/` | pytest suite (fixtures in `tests/conftest.py`); Sphinx docs; MCP host configuration example | + +## 3. Entry points and public interfaces + +- **Python facade** (`import automation_file`): `execute_action`, `execute_files`, + `execute_action_parallel`, `validate_action`, `execute_action_dag`, `add_command_to_executor`, + `executor`, `callback_executor`, `package_manager`, `ActionRegistry`, `build_default_registry`, + `driver_instance` (Google Drive), `start_autocontrol_socket_server`, `start_http_action_server`, + `HTTPActionClient`, `MCPServer`, `create_project_dir`, `launch_ui` (lazy). +- **Action format**: an action is `[name]`, `[name, {kwargs}]` or `[name, [args]]`. A file holds a + list of actions or `{"auto_control": [...]}`. +- **CLI** (`python -m automation_file`; no console script for it): + - legacy flags `-e/--execute_file`, `-d/--execute_dir`, `-c/--create_project` and `--execute_str`. + `_execute_str` decodes a second time when the first `json.loads` yields a string; + - subcommands `zip`, `unzip`, `download`, `create-file`, `server`, `http-server`, `ui`, `mcp`, `drive-upload`. +- **MCP**: `automation_file_mcp` (`automation_file.server.mcp_server:_cli`) or + `python -m automation_file mcp [--allowed-actions ...]`. It is a standard-library JSON-RPC stdio + server whose tools come from the registry (`tools_from_registry`). +- **TCP server**: `start_autocontrol_socket_server(host="localhost", port=9943, allow_non_loopback=False, shared_secret=None, action_acl=None)` + returns a `TCPActionServer`. With a secret, clients prefix the payload with `AUTH \n`. + `quit_server` shuts it down. Replies end with `Return_Data_Over_JE\n`. +- **HTTP server**: `start_http_action_server` returns an `HTTPActionServer` (default `127.0.0.1:9944`) + with `POST /actions` and `GET /healthz`, `/readyz`, `/openapi.json`, `/progress`. Auth is an + optional `Bearer` token. +- **Other servers**: `start_web_ui` (default port 9955) and `start_metrics_server` (`/metrics`, + default port 9945). +- **GUI**: `launch_ui()`, `python -m automation_file ui` or `python main_ui.py`. +- **Plugins**: third-party packages register actions through the entry-point group `automation_file.actions`. + +## 4. Main flows + +**Action list → result** + +``` +JSON file / --execute_str / Python → ActionExecutor.execute_action(list|dict, validate_first, dry_run, substitute) + → _coerce (list or {"auto_control": [...]}) → _execute_event → registry.resolve(name) + → FA_* callable in local/ | remote/ | utils/ | core/ (inside tracing.action_span) + → {"execute: ": return value | repr(error)} (one failure never aborts the batch) +``` + +**Remote transports** + +``` +start_autocontrol_socket_server / start_http_action_server → ensure_loopback (unless allow_non_loopback=True) + → TCP (AUTH + JSON) | HTTP POST /actions (Bearer) → ActionACL check → shared executor +MCP host → automation_file_mcp (stdio JSON-RPC) → tools/call → MCPServer registry (optionally --allowed-actions) +``` + +**Registry construction** + +``` +ActionExecutor() → build_default_registry(): local + http + utils + drive commands + → _register_cloud_backends (register__ops) → trigger / scheduler / progress / notify ops + → _load_plugins (entry points; may override built-ins) + → executor adds FA_execute_action, FA_execute_files, FA_execute_action_parallel, FA_validate +``` + +## 5. Extension points + +- **New local or utility action**: function in `local/_ops.py` (or `utils/`, `core/`) using + `from __future__ import annotations`, `FileAutomationException` subclasses and `file_automation_logger` + → `"FA_"` in `_local_commands()` or `_utils_commands()` (`core/action_registry.py`) → export + from `automation_file/__init__.py` and `__all__` → `tests/test_.py` (plus `tests/test_facade.py` + when exported). +- **New remote backend**, in this order: + 1. `remote//` with `client.py` (module singleton `_instance` with `later_init`, + plus `close` where relevant), the `*_ops.py` modules, and `register__ops(registry)` in `__init__.py`. + 2. Call it from `_register_cloud_backends`. + 3. Add the SDK to `dependencies` in both `stable.toml` and `dev.toml`, then add facade exports. + 4. Add `ui/tabs/_tab.py` and wire it into `ui/tabs/transfer_tab.py`. + 5. Add tests; paths that need the network are not exercised in CI. +- **Outbound HTTP**: always call `validate_http_url` (`remote/url_validator.py`) first. +- **Plugins**: an entry point in the group `automation_file.actions` (`core/plugins.py`), or + `add_command_to_executor({...})` at runtime. `package_manager.add_package_to_executor` registers a + package's members as `_`. +- **CLI subcommand**: `_cmd_` registered in an `_add_*_commands` helper (`__main__.py`). Keep the + legacy flags and the double decode. +- **New server surface**: reuse `ensure_loopback`, `shared_secret` with `hmac.compare_digest`, and `ActionACL`. + +## 6. Cross-project boundaries + +- **PyBreeze (subprocess)** runs `python -m automation_file --execute_str ` or `--execute_file ` + (`PyBreeze/pybreeze/extend/process_executor/python_task_process_manager.py`; the package name is in + `.../process_executor/file_automation/file_automation_process.py`). PyBreeze double-encodes the JSON + on Windows, so `_execute_str`'s `isinstance`-guarded second decode and the legacy flag names are an + implicit contract. PyBreeze also declares `automation-file` as a dependency. +- **TestPioneer** imports `download_file` and `unzip_all` from the facade in-process + (`test_pioneer/executor/file/file_processing.py`). Its `parallel_run` does not spawn this package. +- **Names inherited from AutoControl**: the TCP starter is still called `start_autocontrol_socket_server` + and the action-dict key is `auto_control`; MailThunder uses the same name and key. MailThunder's + socket-server default port (9944) equals this package's HTTP-server default. +- **Wire format**: TCP replies end with the same `Return_Data_Over_JE` terminator as the sibling servers. +- **Builtins policy**: the default registry contains no Python builtins; only `PackageLoader` can add + them. In the siblings, APITestka uses an explicit allowlist, LoadDensity a `_UNSAFE_BUILTINS` + blacklist, and MailThunder registers every builtin (known gap). + +## 7. Design constraints + +- Only the three action shapes in §3. Extend through the registry, not by subclassing the executor. + Python 3.10+, `X | Y` unions, `from __future__ import annotations` (CLAUDE.md § Conventions). +- Exceptions derive from `FileAutomationException`. Log through `file_automation_logger`; no + `print()` diagnostics and no runtime `assert` (§ Conventions; § Code quality › Logging, printing, assertions). +- Every user-supplied URL goes through `validate_http_url`. Never `verify=False`. Downloads have size + and time caps and do not follow redirects (§ Security › Network requests (SSRF prevention) / (TLS)). +- Servers bind loopback unless `allow_non_loopback=True`; secrets are compared with `hmac.compare_digest`; + TCP reads one `recv(8192)` payload; HTTP bodies are capped at 1 MB (§ Security › TCP server; › HTTP server). +- Resolve user paths through `safe_join` / `is_within` (§ Security › Path traversal). SFTP keeps + `paramiko.RejectPolicy()` (§ Security › SFTP host verification). +- `retry_on_transient` retries only the listed exception types (§ Security › Reliability (retry / quota)). + `PackageLoader` is eval-grade; never expose it remotely (§ Security › Plugin / package loading). +- No `shell=True`; subprocesses use argument lists and a timeout (§ Security › General rules; › Subprocess execution). +- Backends and PySide6 are first-class runtime dependencies. Keep `stable.toml` and `dev.toml` + dependencies in sync, and let the publish workflow bump versions (§ Branching & CI). +- Limits: cyclomatic complexity ≤ 15 (hard cap 20), cognitive complexity ≤ 15, functions ≤ 75 lines, + ≤ 7 parameters, nesting ≤ 4, files ≤ 1000 lines (§ Code quality › Complexity & size). +- Run `ruff check`, `ruff format --check`, `mypy` and `pytest` before committing (§ Development). + Development PRs target `dev`; stable PRs target `main` (§ Commit & PR rules). + +## 8. When to update this file + +- A top-level subpackage, backend or server module is added, removed or renamed. +- CLI flags, subcommands, `[project.scripts]` in the TOMLs, or the entry-point group change. +- The action format, the `auto_control` key, the registry build order, or plugin override semantics change. +- Server defaults (host, port, auth, ACL, terminator) or HTTP routes change. +- A §6 contract changes: PyBreeze invocation, the Windows double decode, the facade names TestPioneer uses. +- A CLAUDE.md section referenced in §7 is renamed or its rule changes. +- Refresh the "Last verified" line whenever this file is re-checked against HEAD. diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md new file mode 100644 index 0000000..d4552e2 --- /dev/null +++ b/docs/updates/2026-09.md @@ -0,0 +1,11 @@ +# 2026-09 update log + +Index and query commands: [README.md](README.md). New entries go at the end. + +--- + +## U-20260922-01 · 2026-09-22 · Adopt progress/architecture/docs-updates rules · #docs #migration + +- **What**: `CLAUDE.md` gained the workspace rule section (commit at every stage; `progress.md` holds open items only; finished work is recorded in `docs/updates/`; `architecture.md` is the short overview). Added `architecture.md` and this `docs/updates/` index. `progress.md` starts with the open items found in the workspace inventory of 2026-09-22 (`D:\Codes\docs\updates\2026-09.md`, U-20260922-01..05). +- **Files**: `CLAUDE.md`, `progress.md`, `architecture.md`, `docs/updates/`. +- **Open items**: see `progress.md`. diff --git a/docs/updates/README.md b/docs/updates/README.md new file mode 100644 index 0000000..27a841e --- /dev/null +++ b/docs/updates/README.md @@ -0,0 +1,63 @@ +# docs/updates: update log index + +`progress.md` holds only work that is **not done yet**. Everything that *was* done (what changed, measured numbers, decisions, snapshots) is recorded here: **one batch file per month**, one entry per piece of work, each entry with a fixed-format ID and tags, and one row per entry in the index below. + +> No TODOs here. If an entry mentions something still open, it only points to it (e.g. "open item: `progress.md` #3"); the item itself lives in `progress.md`. + +## How to query + +Run from the repository root: + +| To find | Command | +|---|---| +| every entry, one line each | `rg -n "^## U-2" docs/updates` | +| entries of one type | `rg -n "^## U-2.*#done" docs/updates` | +| entries with a topic tag | `rg -n "^## U-2.*#" docs/updates` | +| one day or one month | `rg -n "^## U-202609" docs/updates` | +| the full text of one entry | `rg -n -A 60 "^## U-20260922-01" docs/updates` | +| any keyword | `rg -n "keyword" docs/updates` | + +Without `rg`: `git grep -n "^## U-2" -- docs/updates`, or in PowerShell `Select-String -Path docs/updates/*.md -Pattern '^## U-2'`. + +## Entry format + +```markdown +## U-YYYYMMDD-NN · YYYY-MM-DD · one-line title · #type #topic + +- **What**: ... +- **Result / numbers**: ... +- **Files**: `path` ... +- **Evidence**: commit, file:line, link ... +- **Open items**: none / see `progress.md` ... +``` + +- **ID**: `U-` + date + two-digit sequence for that day. IDs are never renumbered or reused, so code comments and other documents can cite them. +- **Type tag** (exactly one): `#done` finished `progress.md` item, `#snapshot` measurement or inventory, `#decision`, `#incident`, `#migration`, `#docs`, `#release`. +- Topic tags are free-form (`#mcp`, `#wayland`, ...). +- Keep conclusions, numbers, files and evidence; drop the reasoning trail and dead ends. + +## Batch rules + +1. One file per month: `docs/updates/YYYY-MM.md`. Append new entries at the end. +2. Over about 800 lines, continue in `YYYY-MM-b.md` (then `-c`) and list it in the batch table below. +3. **Claim the ID first**: write the heading line and the index row, then fill in the body. Check the day's last number with `rg -n "^## U-YYYYMMDD" docs/updates`. +4. **One line per index row**: title only (about 60 characters), no summary. +5. Never rewrite a recorded entry. Correct it with a new `#decision` or `#incident` entry and add "→ corrected in U-..." to the old one. + +## When a `progress.md` item is done + +In the same commit: delete the item from `progress.md`, add a `#done` entry here that names it, and add its index row. + +--- + +## Index (newest first) + +| ID | Date | Title | Tags | Batch | +|---|---|---|---|---| +| U-20260922-01 | 2026-09-22 | Adopt progress/architecture/docs-updates rules | #docs #migration | [2026-09](2026-09.md) | + +## Batches + +| File | Period | Entries | +|---|---|---:| +| [2026-09.md](2026-09.md) | 2026-09 | 1 | diff --git a/progress.md b/progress.md new file mode 100644 index 0000000..b7a2564 --- /dev/null +++ b/progress.md @@ -0,0 +1,13 @@ +# progress.md: FileAutomation + +Outstanding work only. When an item is done, delete it in the same commit and add a `#done` entry to `docs/updates/` (format and query commands: `docs/updates/README.md`). No finished items, no history, no rules (rules live in `CLAUDE.md`). +Item numbers (`#n`) are never reused. Tags: [DECIDE] needs the owner's decision, [BLOCKED] waits on something else, [UNVERIFIED] observed but not confirmed. +Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-11, X-16, W-5). + +## Open + +- **#1** Add `.codacy_tmp/` to `.gitignore` (five analysis scratch files sit untracked in the tree). +- **#2** `CLAUDE.md` is stale: ≈:99 says CI runs Python 3.10–3.12 (it runs 3.10–3.14), and the Architecture section does not mention WebDAV, SMB, the MCP server, the DAG runner or notify. +- **#3** Five dependabot branches are unmerged (boxsdk, cryptography, msal, opentelemetry-sdk, pyarrow; 2026-06-01..24), as is `origin/release/bump-v0.0.32` (workspace X-16). +- **#4** Local housekeeping: four local branches are already merged into `dev` (`codacy-fixes-all-issues`, `feat/new-ops-and-backends`, `fix/action-executor-keys`, `fix/codacy-bandit-test-secret`) and local `main` is behind `origin/main` (workspace W-5). +- **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default. `CLAUDE.md` still says "nine tabs" and that the HTTP server handles only `POST /actions`; `architecture.md` describes the code as it is. From 0f433c1308da5e656fcd8e103190f3c54ee8c7fb Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 16:19:50 +0800 Subject: [PATCH 02/20] Stop tracking .idea IDE settings --- .idea/.gitignore | 3 --- .idea/FileAutomation.iml | 11 ----------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 7 ------- .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ docs/updates/2026-09.md | 6 ++++++ docs/updates/README.md | 3 ++- 8 files changed, 8 insertions(+), 42 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/FileAutomation.iml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 26d3352..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml diff --git a/.idea/FileAutomation.iml b/.idea/FileAutomation.iml deleted file mode 100644 index 91f2d42..0000000 --- a/.idea/FileAutomation.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 2cefeb6..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 705b883..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index d4552e2..2bdb43e 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -9,3 +9,9 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What**: `CLAUDE.md` gained the workspace rule section (commit at every stage; `progress.md` holds open items only; finished work is recorded in `docs/updates/`; `architecture.md` is the short overview). Added `architecture.md` and this `docs/updates/` index. `progress.md` starts with the open items found in the workspace inventory of 2026-09-22 (`D:\Codes\docs\updates\2026-09.md`, U-20260922-01..05). - **Files**: `CLAUDE.md`, `progress.md`, `architecture.md`, `docs/updates/`. - **Open items**: see `progress.md`. + +## U-20260922-02 · 2026-09-22 · Stop tracking .idea/ · #done #housekeeping + +- **What**: removed the JetBrains settings under `.idea/` from version control with `git rm -r --cached .idea` (6 files); the files stay on disk. Workspace item W-4 (`D:\Codes\progress.md`). +- **Files**: `.idea/` (untracked); `.gitignore` already ignored `.idea/`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index 27a841e..8d9e633 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -54,10 +54,11 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260922-02 | 2026-09-22 | Stop tracking .idea/ | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-01 | 2026-09-22 | Adopt progress/architecture/docs-updates rules | #docs #migration | [2026-09](2026-09.md) | ## Batches | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 1 | +| [2026-09.md](2026-09.md) | 2026-09 | 2 | From 9e40b3c377467f31cbde84060742ea2c22b3b0f3 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 16:27:32 +0800 Subject: [PATCH 03/20] Record the deleted merged local branches --- docs/updates/2026-09.md | 6 ++++++ docs/updates/README.md | 3 ++- progress.md | 3 +-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 2bdb43e..11ca91c 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -15,3 +15,9 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What**: removed the JetBrains settings under `.idea/` from version control with `git rm -r --cached .idea` (6 files); the files stay on disk. Workspace item W-4 (`D:\Codes\progress.md`). - **Files**: `.idea/` (untracked); `.gitignore` already ignored `.idea/`. - **Open items**: none. + +## U-20260922-03 · 2026-09-22 · Delete four merged local branches and fast-forward main · #done #housekeeping + +- **What**: deleted the local branches `codacy-fixes-all-issues` (9b2531f), `feat/new-ops-and-backends` (55d2041), `fix/action-executor-keys` (d1b4f20) and `fix/codacy-bandit-test-secret` (c1a43c2); each is an ancestor of `origin/main` and its PR (#57, #59, #60, #61, #63) was merged. Local `main` was fast-forwarded to `origin/main` (it was 3 behind). Closes `progress.md` #4 (workspace W-5). The remote branches are untouched. +- **Files**: `progress.md`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index 8d9e633..1c7c48f 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -54,6 +54,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260922-03 | 2026-09-22 | Delete four merged local branches and fast-forward main | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-02 | 2026-09-22 | Stop tracking .idea/ | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-01 | 2026-09-22 | Adopt progress/architecture/docs-updates rules | #docs #migration | [2026-09](2026-09.md) | @@ -61,4 +62,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 2 | +| [2026-09.md](2026-09.md) | 2026-09 | 3 | diff --git a/progress.md b/progress.md index b7a2564..f1ed5da 100644 --- a/progress.md +++ b/progress.md @@ -2,12 +2,11 @@ Outstanding work only. When an item is done, delete it in the same commit and add a `#done` entry to `docs/updates/` (format and query commands: `docs/updates/README.md`). No finished items, no history, no rules (rules live in `CLAUDE.md`). Item numbers (`#n`) are never reused. Tags: [DECIDE] needs the owner's decision, [BLOCKED] waits on something else, [UNVERIFIED] observed but not confirmed. -Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-11, X-16, W-5). +Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-11, X-16). ## Open - **#1** Add `.codacy_tmp/` to `.gitignore` (five analysis scratch files sit untracked in the tree). - **#2** `CLAUDE.md` is stale: ≈:99 says CI runs Python 3.10–3.12 (it runs 3.10–3.14), and the Architecture section does not mention WebDAV, SMB, the MCP server, the DAG runner or notify. - **#3** Five dependabot branches are unmerged (boxsdk, cryptography, msal, opentelemetry-sdk, pyarrow; 2026-06-01..24), as is `origin/release/bump-v0.0.32` (workspace X-16). -- **#4** Local housekeeping: four local branches are already merged into `dev` (`codacy-fixes-all-issues`, `feat/new-ops-and-backends`, `fix/action-executor-keys`, `fix/codacy-bandit-test-secret`) and local `main` is behind `origin/main` (workspace W-5). - **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default. `CLAUDE.md` still says "nine tabs" and that the HTTP server handles only `POST /actions`; `architecture.md` describes the code as it is. From aaa4337fa495e3037eb582a77cfcbb19485c2a18 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 16:28:06 +0800 Subject: [PATCH 04/20] Point the homepage at the FileAutomation repository and unify the author email --- dev.toml | 4 ++-- docs/updates/2026-09.md | 6 ++++++ docs/updates/README.md | 3 ++- progress.md | 2 +- stable.toml | 4 ++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dev.toml b/dev.toml index ec2af07..e0c40d2 100644 --- a/dev.toml +++ b/dev.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" name = "automation_file_dev" version = "0.0.33" authors = [ - { name = "JE-Chen", email = "zenmailman@gmail.com" }, + { name = "JE-Chen", email = "jechenmailman@gmail.com" }, ] description = "JSON-driven file, Drive, and cloud automation framework (dev channel)." readme = { file = "README.md", content-type = "text/markdown" } @@ -60,7 +60,7 @@ dev = [ automation_file_mcp = "automation_file.server.mcp_server:_cli" [project.urls] -"Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" +"Homepage" = "https://github.com/Integration-Automation/FileAutomation" [tool.setuptools.packages] find = { namespaces = false } diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 11ca91c..109405c 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -21,3 +21,9 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What**: deleted the local branches `codacy-fixes-all-issues` (9b2531f), `feat/new-ops-and-backends` (55d2041), `fix/action-executor-keys` (d1b4f20) and `fix/codacy-bandit-test-secret` (c1a43c2); each is an ancestor of `origin/main` and its PR (#57, #59, #60, #61, #63) was merged. Local `main` was fast-forwarded to `origin/main` (it was 3 behind). Closes `progress.md` #4 (workspace W-5). The remote branches are untouched. - **Files**: `progress.md`. - **Open items**: none. + +## U-20260922-04 · 2026-09-22 · Point project URLs at the current repository · #done #metadata + +- **What**: replaced the misspelled or renamed GitHub owner in the project URLs (`Intergration-Automation-Testing/…`, and where present `JE-Chen/je_editor`, `JE-Chen/Integration-testing-environment`, the `AutoControl`/`AutomationEditor`/`AutomationFile`/`JEditor` repo names) with the current `origin`. The `Homepage` in `stable.toml` and `dev.toml` pointed at `JE-Chen/Integration-testing-environment`; the author email was `zenmailman@gmail.com` while every other workspace package uses `jechenmailman@gmail.com`. Workspace item X-11. +- **Files**: `dev.toml`, `stable.toml`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index 1c7c48f..a064f85 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -54,6 +54,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260922-04 | 2026-09-22 | Point project URLs at the current repository | #done #metadata | [2026-09](2026-09.md) | | U-20260922-03 | 2026-09-22 | Delete four merged local branches and fast-forward main | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-02 | 2026-09-22 | Stop tracking .idea/ | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-01 | 2026-09-22 | Adopt progress/architecture/docs-updates rules | #docs #migration | [2026-09](2026-09.md) | @@ -62,4 +63,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 3 | +| [2026-09.md](2026-09.md) | 2026-09 | 4 | diff --git a/progress.md b/progress.md index f1ed5da..37d2b8e 100644 --- a/progress.md +++ b/progress.md @@ -2,7 +2,7 @@ Outstanding work only. When an item is done, delete it in the same commit and add a `#done` entry to `docs/updates/` (format and query commands: `docs/updates/README.md`). No finished items, no history, no rules (rules live in `CLAUDE.md`). Item numbers (`#n`) are never reused. Tags: [DECIDE] needs the owner's decision, [BLOCKED] waits on something else, [UNVERIFIED] observed but not confirmed. -Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-11, X-16). +Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-16). ## Open diff --git a/stable.toml b/stable.toml index e60cad9..81667b5 100644 --- a/stable.toml +++ b/stable.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" name = "automation_file" version = "0.0.31" authors = [ - { name = "JE-Chen", email = "zenmailman@gmail.com" }, + { name = "JE-Chen", email = "jechenmailman@gmail.com" }, ] description = "JSON-driven file, Drive, and cloud automation framework." readme = { file = "README.md", content-type = "text/markdown" } @@ -60,7 +60,7 @@ dev = [ automation_file_mcp = "automation_file.server.mcp_server:_cli" [project.urls] -"Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" +"Homepage" = "https://github.com/Integration-Automation/FileAutomation" [tool.setuptools.packages] find = { namespaces = false } From 8e8b94f38c802f8729c8a67454551592da0b0943 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 16:55:43 +0800 Subject: [PATCH 05/20] Guard the legacy CLI flags other repositories call --- architecture.md | 5 +- docs/updates/2026-09.md | 7 +++ docs/updates/README.md | 3 +- progress.md | 2 +- tests/test_legacy_cli_contract.py | 90 +++++++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 tests/test_legacy_cli_contract.py diff --git a/architecture.md b/architecture.md index f08b4d5..5138724 100644 --- a/architecture.md +++ b/architecture.md @@ -113,8 +113,9 @@ ActionExecutor() → build_default_registry(): local + http + utils + drive comm - **PyBreeze (subprocess)** runs `python -m automation_file --execute_str ` or `--execute_file ` (`PyBreeze/pybreeze/extend/process_executor/python_task_process_manager.py`; the package name is in `.../process_executor/file_automation/file_automation_process.py`). PyBreeze double-encodes the JSON - on Windows, so `_execute_str`'s `isinstance`-guarded second decode and the legacy flag names are an - implicit contract. PyBreeze also declares `automation-file` as a dependency. + on Windows, so `_execute_str`'s `isinstance`-guarded second decode and the legacy flag names are a + contract, guarded by `tests/test_legacy_cli_contract.py`. PyBreeze also declares `automation-file` as + a dependency. - **TestPioneer** imports `download_file` and `unzip_all` from the facade in-process (`test_pioneer/executor/file/file_processing.py`). Its `parallel_run` does not spawn this package. - **Names inherited from AutoControl**: the TCP starter is still called `start_autocontrol_socket_server` diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 109405c..3e26f12 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -27,3 +27,10 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What**: replaced the misspelled or renamed GitHub owner in the project URLs (`Intergration-Automation-Testing/…`, and where present `JE-Chen/je_editor`, `JE-Chen/Integration-testing-environment`, the `AutoControl`/`AutomationEditor`/`AutomationFile`/`JEditor` repo names) with the current `origin`. The `Homepage` in `stable.toml` and `dev.toml` pointed at `JE-Chen/Integration-testing-environment`; the author email was `zenmailman@gmail.com` while every other workspace package uses `jechenmailman@gmail.com`. Workspace item X-11. - **Files**: `dev.toml`, `stable.toml`. - **Open items**: none. + +## U-20260922-05 · 2026-09-22 · Contract test for the legacy CLI flags · #done #tests + +- **What**: added `tests/test_legacy_cli_contract.py`. It runs `python -m automation_file` from this checkout as a child process (cwd in `tmp_path`) and checks `-e`/`--execute_file`, `-d`/`--execute_dir`, `--execute_str` encoded exactly as PyBreeze encodes it (a second `json.dumps` on Windows), `-c`/`--create_project`, a non-zero exit with no arguments, and that `create_project_dir` stays importable from the package. The action is `FA_create_file` into `tmp_path`, so a passing run proves it executed. Workspace item X-7. +- **Checks**: the file passes on its own three times in a row (about 80 s, mostly nine interpreter start-ups); full suite 768 passed. One earlier full run failed a single case while AutoControlGUI's editable tree (loaded as a pytest plugin) was mid-edit in another session; it did not recur. ruff clean. +- **Files**: `tests/test_legacy_cli_contract.py`, `architecture.md` §6, `progress.md`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index a064f85..2358645 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -54,6 +54,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260922-05 | 2026-09-22 | Contract test for the legacy CLI flags | #done #tests | [2026-09](2026-09.md) | | U-20260922-04 | 2026-09-22 | Point project URLs at the current repository | #done #metadata | [2026-09](2026-09.md) | | U-20260922-03 | 2026-09-22 | Delete four merged local branches and fast-forward main | #done #housekeeping | [2026-09](2026-09.md) | | U-20260922-02 | 2026-09-22 | Stop tracking .idea/ | #done #housekeeping | [2026-09](2026-09.md) | @@ -63,4 +64,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 4 | +| [2026-09.md](2026-09.md) | 2026-09 | 5 | diff --git a/progress.md b/progress.md index 37d2b8e..00b8e9c 100644 --- a/progress.md +++ b/progress.md @@ -2,7 +2,7 @@ Outstanding work only. When an item is done, delete it in the same commit and add a `#done` entry to `docs/updates/` (format and query commands: `docs/updates/README.md`). No finished items, no history, no rules (rules live in `CLAUDE.md`). Item numbers (`#n`) are never reused. Tags: [DECIDE] needs the owner's decision, [BLOCKED] waits on something else, [UNVERIFIED] observed but not confirmed. -Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-7, X-16). +Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X-6, X-16). ## Open diff --git a/tests/test_legacy_cli_contract.py b/tests/test_legacy_cli_contract.py new file mode 100644 index 0000000..9b79067 --- /dev/null +++ b/tests/test_legacy_cli_contract.py @@ -0,0 +1,90 @@ +"""Contract test for the legacy CLI flags that other repositories call. + +PyBreeze starts ``python -m automation_file --execute_str `` (JSON-encoded a second +time on Windows, see ``pybreeze/extend/process_executor/python_task_process_manager.py``) +and ``--execute_file ``; TestPioneer's ``parallel_run`` starts +``--execute_file ``; PyBreeze's "create project" menu calls +``automation_file.create_project_dir()`` in process. Nothing on the consumer side tests +these, so this file does: renaming or removing a flag, or dropping the second +JSON decode on Windows, breaks them (workspace item X-7). +""" +import json +import os +import subprocess # nosec B404 - the CLI is exercised as a real child process +import sys +from pathlib import Path + +import pytest + +import automation_file + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACKAGE = "automation_file" +IS_WINDOWS = sys.platform in ("win32", "cygwin", "msys") + + +def _run_cli(cwd: Path, *args: str) -> subprocess.CompletedProcess: + """Run ``python -m PACKAGE`` from this checkout; *cwd* catches any log file it writes.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(REPO_ROOT), env.get("PYTHONPATH")])) + env["PYTHONIOENCODING"] = "utf-8" + return subprocess.run( # nosec B603 - fixed interpreter, test-controlled arguments + [sys.executable, "-m", PACKAGE, *args], + cwd=cwd, env=env, capture_output=True, text=True, encoding="utf-8", + timeout=300, check=False, + ) + + +def _pybreeze_execute_str(actions: list) -> str: + """Encode *actions* the way PyBreeze's ``start_test_process`` does.""" + payload = json.dumps(actions) + return json.dumps(payload) if IS_WINDOWS else payload + + +def _actions(target: Path) -> list: + """A harmless action list whose only effect is writing *target*.""" + return [["FA_create_file", {"file_path": str(target), "content": "x"}]] + + +def _assert_ran(result: subprocess.CompletedProcess, target: Path) -> None: + assert result.returncode == 0, result.stderr + assert target.exists(), result.stdout + result.stderr + + +@pytest.mark.parametrize("flag", ["-e", "--execute_file"]) +def test_execute_file(tmp_path, flag): + target = tmp_path / "created.txt" + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps(_actions(target)), encoding="utf-8") + _assert_ran(_run_cli(tmp_path, flag, str(action_file)), target) + + +@pytest.mark.parametrize("flag", ["-d", "--execute_dir"]) +def test_execute_dir(tmp_path, flag): + target = tmp_path / "created.txt" + action_dir = tmp_path / "actions" + action_dir.mkdir() + (action_dir / "actions.json").write_text(json.dumps(_actions(target)), encoding="utf-8") + _assert_ran(_run_cli(tmp_path, flag, str(action_dir)), target) + + +def test_execute_str_as_pybreeze_sends_it(tmp_path): + target = tmp_path / "created.txt" + _assert_ran(_run_cli(tmp_path, "--execute_str", _pybreeze_execute_str(_actions(target))), target) + + +@pytest.mark.parametrize("flag", ["-c", "--create_project"]) +def test_create_project(tmp_path, flag): + project = tmp_path / "project" + result = _run_cli(tmp_path, flag, str(project)) + assert result.returncode == 0, result.stderr + assert project.is_dir(), result.stdout + result.stderr + assert any(project.rglob("*.json")), result.stdout + result.stderr + + +def test_no_flag_exits_non_zero(tmp_path): + assert _run_cli(tmp_path).returncode != 0 + + +def test_create_project_dir_is_exported(): + assert callable(automation_file.create_project_dir) From 849ee1c6749bb8a32a697569a07d343524acd981 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 16:58:15 +0800 Subject: [PATCH 06/20] Claim update-log IDs under an atomic lock Two parallel sessions picked the same entry number because claiming an ID first does not stop simultaneous writers. Entries are now numbered under an atomic mkdir lock in docs/updates/, with a duplicate check before commit. --- docs/updates/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/updates/README.md b/docs/updates/README.md index 2358645..889bfc5 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -40,7 +40,11 @@ Without `rg`: `git grep -n "^## U-2" -- docs/updates`, or in PowerShell `Select- 1. One file per month: `docs/updates/YYYY-MM.md`. Append new entries at the end. 2. Over about 800 lines, continue in `YYYY-MM-b.md` (then `-c`) and list it in the batch table below. -3. **Claim the ID first**: write the heading line and the index row, then fill in the body. Check the day's last number with `rg -n "^## U-YYYYMMDD" docs/updates`. +3. **Claim the ID under a lock.** Several sessions may write this log at the same time (for example parallel autonomous runs), and without a lock two of them pick the same number: + 1. `mkdir docs/updates/.id-lock`. Creating a directory is atomic, so only one writer succeeds. If it already exists, someone else is claiming: wait a few seconds and retry. A lock older than 10 minutes is stale and may be removed. + 2. Find the day's last number with `rg -n "^## U-YYYYMMDD" docs/updates` and write the heading line and the index row. + 3. `rmdir docs/updates/.id-lock`, then fill in the body. Git never tracks the empty lock directory. + 4. Before committing, `rg -c "^## U-" docs/updates` must report one match in total. If not, renumber your entry under the lock and fix its index row. Whoever merges a branch renumbers entries that reuse an ID. 4. **One line per index row**: title only (about 60 characters), no summary. 5. Never rewrite a recorded entry. Correct it with a new `#decision` or `#incident` entry and add "→ corrected in U-..." to the old one. From ce13593fe8685a9bf89659ce2c9741a39ec5b7c9 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 22 Sep 2026 21:54:29 +0800 Subject: [PATCH 07/20] Ignore the local Codacy scratch directory --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c9b638b..831dcb6 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ credentials.json **/credentials.json .claude/ + +# Local Codacy / Sonar scratch output +.codacy_tmp/ From 2a8de7e28be2091d3966567d2011a72522340ab8 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 01:58:09 +0800 Subject: [PATCH 08/20] Raise four dependency floors, keep boxsdk at 3.x and point Dependabot at dev --- .github/dependabot.yml | 3 +++ dev.toml | 8 ++++---- docs/updates/2026-09.md | 9 +++++++++ docs/updates/README.md | 3 ++- progress.md | 3 ++- stable.toml | 8 ++++---- 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba1c6b8..309370b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,5 +7,8 @@ version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests + # Updates belong on dev: main is the released branch and merging into it + # publishes, so a PR opened against main just sits there. + target-branch: "dev" schedule: interval: "daily" diff --git a/dev.toml b/dev.toml index e0c40d2..903b0b2 100644 --- a/dev.toml +++ b/dev.toml @@ -25,14 +25,14 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=47.0.0", + "cryptography>=49.0.0", "prometheus_client>=0.25.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=15.0.0", + "pyarrow>=24.0.0", "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.41.1", - "msal>=1.36.0", + "opentelemetry-sdk>=1.42.1", + "msal>=1.37.0", "boxsdk>=3.14.0,<4", "tomli>=2.0.1; python_version<\"3.11\"" ] diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 3e26f12..11c4e7c 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -34,3 +34,12 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Checks**: the file passes on its own three times in a row (about 80 s, mostly nine interpreter start-ups); full suite 768 passed. One earlier full run failed a single case while AutoControlGUI's editable tree (loaded as a pytest plugin) was mid-edit in another session; it did not recur. ruff clean. - **Files**: `tests/test_legacy_cli_contract.py`, `architecture.md` §6, `progress.md`. - **Open items**: none. + +## U-20260923-01 · 2026-09-23 · Dependency floors raised; Dependabot on dev; boxsdk 10 refused · #done #deps #ci + +- **What** (`progress.md` #3, workspace X-16): the five Dependabot PRs sat unmerged since June because they target `main`, the released branch. Four of them are applied here on `dev` instead, in both `dev.toml` and `stable.toml`: `cryptography>=49.0.0` (#87), `pyarrow>=24.0.0` (#89), `opentelemetry-sdk>=1.42.1` (#88), `msal>=1.37.0` (#83). Those PRs are closed with a pointer to this commit. +- **boxsdk stays at `>=3.14.0,<4`** and PR #86 (`>=10.12.0,<11`) is closed as breaking: the `boxsdk` distribution's 10.x line installs the module **`box_sdk_gen`**, not `boxsdk` (verified by installing `boxsdk==10.15.0` into a scratch venv: `site-packages/box_sdk_gen` only, `import boxsdk` raises `ModuleNotFoundError`). `automation_file/remote/box/client.py` imports `boxsdk` and builds `boxsdk.OAuth2` / `boxsdk.Client`, so the cap is what keeps the Box backend working. Porting it to the new SDK is now `progress.md` #6. +- **Dependabot**: `.github/dependabot.yml` sets `target-branch: "dev"`, so future updates land where work happens and ship with the next release. +- **Checks**: `pytest tests` 752 passed, 8 skipped (metadata-only change; the optional extras are not installed locally). +- **Files**: `dev.toml`, `stable.toml`, `.github/dependabot.yml`, `progress.md`. +- **Open items**: `progress.md` #6 (Box SDK port). `origin/release/bump-v0.0.32` is still unmerged, which is the rest of #3. diff --git a/docs/updates/README.md b/docs/updates/README.md index 889bfc5..78b6408 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-01 | 2026-09-23 | Dependency floors raised; Dependabot on dev; boxsdk 10 refused | #done #deps #ci | [2026-09](2026-09.md) | | U-20260922-05 | 2026-09-22 | Contract test for the legacy CLI flags | #done #tests | [2026-09](2026-09.md) | | U-20260922-04 | 2026-09-22 | Point project URLs at the current repository | #done #metadata | [2026-09](2026-09.md) | | U-20260922-03 | 2026-09-22 | Delete four merged local branches and fast-forward main | #done #housekeeping | [2026-09](2026-09.md) | @@ -68,4 +69,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 5 | +| [2026-09.md](2026-09.md) | 2026-09 | 6 | diff --git a/progress.md b/progress.md index 00b8e9c..0563321 100644 --- a/progress.md +++ b/progress.md @@ -8,5 +8,6 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#1** Add `.codacy_tmp/` to `.gitignore` (five analysis scratch files sit untracked in the tree). - **#2** `CLAUDE.md` is stale: ≈:99 says CI runs Python 3.10–3.12 (it runs 3.10–3.14), and the Architecture section does not mention WebDAV, SMB, the MCP server, the DAG runner or notify. -- **#3** Five dependabot branches are unmerged (boxsdk, cryptography, msal, opentelemetry-sdk, pyarrow; 2026-06-01..24), as is `origin/release/bump-v0.0.32` (workspace X-16). +- **#3** `origin/release/bump-v0.0.32` is still unmerged (workspace X-16). The five dependabot branches are settled: four floors applied on `dev` and boxsdk refused, see `docs/updates` U-20260923-01. - **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default. `CLAUDE.md` still says "nine tabs" and that the HTTP server handles only `POST /actions`; `architecture.md` describes the code as it is. +- **#6** The Box backend (`automation_file/remote/box/client.py`) uses the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), and the distribution's 10.x line ships the module `box_sdk_gen` with a different API, so the `>=3.14.0,<4` cap is load-bearing. Port the client to `box_sdk_gen` (and drop the cap), or write down that the Box backend stays on the 3.x SDK. diff --git a/stable.toml b/stable.toml index 81667b5..c2c4820 100644 --- a/stable.toml +++ b/stable.toml @@ -25,14 +25,14 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=47.0.0", + "cryptography>=49.0.0", "prometheus_client>=0.25.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=15.0.0", + "pyarrow>=24.0.0", "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.41.1", - "msal>=1.36.0", + "opentelemetry-sdk>=1.42.1", + "msal>=1.37.0", "boxsdk>=3.14.0,<4", "tomli>=2.0.1; python_version<\"3.11\"" ] From af7733efa9e3fb072da9851905860accd678f73d Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 05:11:17 +0800 Subject: [PATCH 09/20] Bring CLAUDE.md in line with the package, CI and HTTP routes --- CLAUDE.md | 92 ++++++++++++++--------------------------- docs/updates/2026-09.md | 11 +++++ docs/updates/README.md | 3 +- progress.md | 4 +- 4 files changed, 46 insertions(+), 64 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 08006e2..787c2ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,69 +8,41 @@ Automation-first Python library for local file / directory / zip operations, HTT ``` automation_file/ -├── __init__.py # Public API facade (every name users import) -├── __main__.py # CLI entry (argparse dispatcher, subcommands + legacy flags) -├── exceptions.py # Exception hierarchy (FileAutomationException base) -├── logging_config.py # file_automation_logger (file + stderr handlers) -├── core/ -│ ├── action_registry.py # ActionRegistry — name -> callable (Registry + Command) -│ ├── action_executor.py # ActionExecutor — runs JSON action lists (Facade + Template Method) -│ ├── callback_executor.py # CallbackExecutor — trigger then callback composition -│ ├── package_loader.py # PackageLoader — dynamically registers package members -│ ├── json_store.py # Thread-safe read/write of JSON action files -│ ├── retry.py # retry_on_transient — capped exponential back-off decorator -│ └── quota.py # Quota — size + time budget guards -├── local/ # Strategy modules — each file is a batch of pure operations -│ ├── file_ops.py -│ ├── dir_ops.py -│ ├── zip_ops.py -│ └── safe_paths.py # safe_join / is_within — path traversal guard -├── remote/ -│ ├── url_validator.py # SSRF guard for outbound URLs -│ ├── http_download.py # SSRF-validated HTTP download with size/timeout caps + retry -│ ├── google_drive/ -│ │ ├── client.py # GoogleDriveClient (Singleton Facade) -│ │ ├── delete_ops.py -│ │ ├── download_ops.py -│ │ ├── folder_ops.py -│ │ ├── search_ops.py -│ │ ├── share_ops.py -│ │ └── upload_ops.py -│ ├── s3/ # S3 (boto3) — auto-registered in build_default_registry() -│ │ ├── client.py # S3Client -│ │ ├── upload_ops.py -│ │ ├── download_ops.py -│ │ ├── delete_ops.py -│ │ └── list_ops.py -│ ├── azure_blob/ # Azure Blob — auto-registered in build_default_registry() -│ │ └── {client,upload,download,delete,list}_ops.py -│ ├── dropbox_api/ # Dropbox — auto-registered in build_default_registry() -│ │ └── {client,upload,download,delete,list}_ops.py -│ └── sftp/ # SFTP (paramiko + RejectPolicy) — auto-registered in build_default_registry() -│ └── {client,upload,download,delete,list}_ops.py -├── server/ -│ ├── tcp_server.py # Loopback-only TCP server executing JSON actions (optional shared-secret auth) -│ └── http_server.py # Loopback-only HTTP server (POST /actions, optional Bearer auth) -├── project/ -│ ├── project_builder.py # ProjectBuilder (Builder pattern) -│ └── templates.py # Scaffolding templates -├── ui/ # PySide6 GUI (required dep) -│ ├── launcher.py # launch_ui(argv) — boots QApplication + MainWindow -│ ├── main_window.py # MainWindow — tabbed control surface over every feature -│ ├── worker.py # ActionWorker(QRunnable) + _WorkerSignals -│ ├── log_widget.py # LogPanel — timestamped, read-only log stream -│ └── tabs/ # One tab per domain: local / http / drive / s3 / -│ # azure / dropbox / sftp / -│ # JSON actions / servers -└── utils/ - └── file_discovery.py # Recursive file listing by extension +├── __init__.py # Public API facade (__all__); launch_ui is loaded lazily via __getattr__ +├── __main__.py # CLI entry: subcommands plus the legacy -e/-d/-c/--execute_str flags +├── exceptions.py # FileAutomationException hierarchy +├── logging_config.py # file_automation_logger (file + stderr handlers) +├── core/ # Engine: action_registry (ActionRegistry, build_default_registry), action_executor +│ # (shared `executor`), callback_executor, package_loader, plugins, dag_executor, +│ # action_queue, json_store, substitution; cross-cutting helpers: retry, quota, +│ # rate_limit, circuit_breaker, file_lock, sqlite_lock, checksum, manifest, crypto, +│ # secrets, config, config_watcher, audit, metrics, tracing, progress, fim, content_store +├── local/ # Strategy modules: file/dir/zip/tar/archive ops, sync, diff, text/JSON/data edits, +│ # templates, versioning, trash, shell_ops (argv-only subprocess), conditional; +│ # safe_paths.py guards against path traversal +├── remote/ # url_validator (SSRF guard), http_download, cross_backend, fsspec_bridge, and one +│ # subpackage per backend: google_drive, s3, azure_blob, dropbox_api, sftp, ftp, +│ # onedrive, box (client.py + *_ops.py + register__ops); smb and webdav +│ # have a client only +├── server/ # tcp_server, http_server, mcp_server (MCP over stdio), web_ui, metrics_server, +│ # action_acl (ActionACL), network_guards (ensure_loopback) +├── client/ # HTTPActionClient for the HTTP action server +├── trigger/, scheduler/, notify/ # watchdog file triggers, cron scheduler, notification sinks; +│ # each registers its own FA_* ops +├── project/ # ProjectBuilder, create_project_dir +├── ui/ # PySide6 GUI: launcher.launch_ui, main_window.MainWindow, worker.ActionWorker, +│ # log_widget.LogPanel, tabs/ (home, local, http, JSON editor, servers, scheduler, +│ # trigger, progress; the cloud backends are panels grouped under transfer_tab) +└── utils/ # file discovery, fast find, grep, duplicate finder, backup rotation ``` +`architecture.md` §2 carries the same map with one row per directory; keep the two in step. + **Key design patterns in use:** - **Facade**: `automation_file/__init__.py` re-exports every supported name (`execute_action`, `driver_instance`, `start_autocontrol_socket_server`, …). - **Registry + Command**: `ActionRegistry` maps action name → callable. JSON action lists are command objects (`[name, kwargs]` / `[name, [args]]` / `[name]`) dispatched through the registry. - **Template Method**: `ActionExecutor._execute_event` defines the single-action lifecycle (resolve → call → wrap result); `execute_action` is the outer iteration template. -- **Strategy**: Each `local/*_ops.py` and `remote/google_drive/*_ops.py` module is an independent strategy that plugs into the registry. +- **Strategy**: Each `local/*_ops.py` and `remote//*_ops.py` module is an independent strategy that plugs into the registry. - **Singleton (module-level)**: `driver_instance`, `executor`, `callback_executor`, `package_manager` are shared instances wired in `__init__.py` so `callback_executor.registry is executor.registry`. - **Builder**: `ProjectBuilder` assembles the `keyword/` + `executor/` skeleton. @@ -85,7 +57,7 @@ automation_file/ - `MainWindow` — PySide6 tabbed control surface (`ui/main_window.py`). Nine tabs — Local, HTTP, Google Drive, S3, Azure Blob, Dropbox, SFTP, JSON actions, Servers — share a `LogPanel` and dispatch work through `ActionWorker(QRunnable)` on the global `QThreadPool`. - `launch_ui(argv=None)` — boots / reuses a `QApplication`, shows `MainWindow`, and returns the exec code. Exposed lazily on the facade via `__getattr__` so the Qt runtime isn't paid for by non-UI importers. - `TCPActionServer` — threaded TCP server that deserialises a JSON action list per connection. Defaults to loopback; optional `shared_secret` enforces `AUTH \n` prefix. -- `HTTPActionServer` — `ThreadingHTTPServer` exposing `POST /actions`. Defaults to loopback; optional `shared_secret` enforces `Authorization: Bearer `. +- `HTTPActionServer` — `ThreadingHTTPServer` exposing `POST /actions` plus `GET /healthz`, `/readyz`, `/openapi.json` and `/progress`. Defaults to loopback; optional `shared_secret` enforces `Authorization: Bearer `. - `Quota` — frozen dataclass capping bytes and wall-clock seconds per action or block (`check_size`, `time_budget` context manager, `wraps` decorator). `0` disables each cap. - `retry_on_transient(max_attempts, backoff_base, backoff_cap, retriable)` — decorator that retries with capped exponential back-off and raises `RetryExhaustedException` chained to the last error. - `safe_join(root, user_path)` / `is_within(root, path)` — path traversal guard; `safe_join` raises `PathTraversalException` when the resolved path escapes `root`. @@ -96,7 +68,7 @@ automation_file/ - `dev` branch: development, publishes `automation_file_dev` to PyPI (version in `dev.toml`). - Keep `dependencies` and `[project.optional-dependencies]` (`dev`) in sync across both TOMLs. Backends (`boto3`, `azure-storage-blob`, `dropbox`, `paramiko`) and `PySide6` are first-class runtime deps — do not move them back under extras. - **Version bumping is automatic.** A dedicated publish workflow bumps the patch in both `stable.toml` and `dev.toml`, builds, uploads to PyPI, then commits the bump back to `main` tagged as `vX.Y.Z`. Do not hand-bump before merging to `main`. The next publish run is skipped via a commit-message guard (`chore: bump version`), so the bump itself never re-triggers publishing. -- CI: GitHub Actions (Windows, Python 3.10 / 3.11 / 3.12) — one matrix workflow per branch: `.github/workflows/ci-dev.yml`, `.github/workflows/ci-stable.yml`. +- CI: GitHub Actions — a `lint` job on Ubuntu (Python 3.12), then `pytest` on Windows across Python 3.10 / 3.11 / 3.12 / 3.13 / 3.14. One workflow per branch: `.github/workflows/ci-dev.yml`, `.github/workflows/ci-stable.yml`. - CI steps: `lint` (ruff check + ruff format --check + mypy) → `pytest` with coverage → uploads `coverage.xml` as an artifact. - Publishing lives in a separate workflow (`.github/workflows/publish.yml`) that runs on push to `main`: bumps both TOMLs, copies `stable.toml` to `pyproject.toml`, builds the sdist + wheel, `twine upload` via `PYPI_API_TOKEN`, then commits + tags + pushes and creates `gh release create v --generate-notes`. - `pre-commit` is configured (`.pre-commit-config.yaml`): trailing-whitespace, eof-fixer, check-yaml, check-toml, check-added-large-files, ruff, ruff-format, mypy. Install with `pre-commit install` after cloning. @@ -163,7 +135,7 @@ All code must follow secure-by-default principles. Review every change against t ### HTTP server - `HTTPActionServer` / `start_http_action_server` mirror the TCP server's posture: loopback-only by default, `allow_non_loopback=True` required to bind elsewhere, optional `shared_secret` enforced as `Authorization: Bearer ` using `hmac.compare_digest`. -- Only `POST /actions` is handled. Request body capped at 1 MB — do not raise without also switching to a streaming parser. +- `POST /actions` is the only endpoint that runs anything; the `GET` routes (`/healthz`, `/readyz`, `/openapi.json`, `/progress`) only report. Request body capped at 1 MB — do not raise without also switching to a streaming parser. - Responses are JSON. Auth failures return `401`; malformed JSON returns `400`; unknown paths return `404`. ### Path traversal diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 11c4e7c..8b2ef0b 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -43,3 +43,14 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Checks**: `pytest tests` 752 passed, 8 skipped (metadata-only change; the optional extras are not installed locally). - **Files**: `dev.toml`, `stable.toml`, `.github/dependabot.yml`, `progress.md`. - **Open items**: `progress.md` #6 (Box SDK port). `origin/release/bump-v0.0.32` is still unmerged, which is the rest of #3. + +## U-20260923-02 · 2026-09-23 · CLAUDE.md matches the code again · #done #docs + +- **What** (`progress.md` #1, #2 and the documentation half of #5): + - `CLAUDE.md`'s package tree listed a 2026-04 layout: no `ftp/`, `onedrive/`, `box/`, `smb/`, `webdav/`, no MCP server, web UI or metrics server, no `client/`, `trigger/`, `scheduler/`, `notify/`, and a nine-tab GUI that has since become home / local / HTTP / JSON editor / servers / scheduler / trigger / progress with the cloud backends grouped under the transfer tab. It is replaced by a compact map that matches `architecture.md` §2, with a line asking to keep the two in step. + - CI is described as it runs: a `lint` job on Ubuntu (Python 3.12), then `pytest` on Windows across 3.10–3.14 (it said 3.10–3.12). + - The HTTP server section said only `POST /actions` is handled; it also serves `GET /healthz`, `/readyz`, `/openapi.json` and `/progress`, which only report. + - The Strategy bullet now names every `remote//*_ops.py`, not just Google Drive. + - `.codacy_tmp/` was already ignored by `ce13593` (#1); the item just had not been removed. +- **Files**: `CLAUDE.md`, `progress.md`. +- **Open items**: `progress.md` #5 keeps only the port clash (9944 is also MailThunder's socket-server default). diff --git a/docs/updates/README.md b/docs/updates/README.md index 78b6408..3e9e37e 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-02 | 2026-09-23 | CLAUDE.md matches the code again | #done #docs | [2026-09](2026-09.md) | | U-20260923-01 | 2026-09-23 | Dependency floors raised; Dependabot on dev; boxsdk 10 refused | #done #deps #ci | [2026-09](2026-09.md) | | U-20260922-05 | 2026-09-22 | Contract test for the legacy CLI flags | #done #tests | [2026-09](2026-09.md) | | U-20260922-04 | 2026-09-22 | Point project URLs at the current repository | #done #metadata | [2026-09](2026-09.md) | @@ -69,4 +70,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 6 | +| [2026-09.md](2026-09.md) | 2026-09 | 7 | diff --git a/progress.md b/progress.md index 0563321..1d58230 100644 --- a/progress.md +++ b/progress.md @@ -6,8 +6,6 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- ## Open -- **#1** Add `.codacy_tmp/` to `.gitignore` (five analysis scratch files sit untracked in the tree). -- **#2** `CLAUDE.md` is stale: ≈:99 says CI runs Python 3.10–3.12 (it runs 3.10–3.14), and the Architecture section does not mention WebDAV, SMB, the MCP server, the DAG runner or notify. - **#3** `origin/release/bump-v0.0.32` is still unmerged (workspace X-16). The five dependabot branches are settled: four floors applied on `dev` and boxsdk refused, see `docs/updates` U-20260923-01. -- **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default. `CLAUDE.md` still says "nine tabs" and that the HTTP server handles only `POST /actions`; `architecture.md` describes the code as it is. +- **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default, so the two cannot run side by side on their defaults. Pick a free default for one of them (and note it in both `architecture.md` §6). - **#6** The Box backend (`automation_file/remote/box/client.py`) uses the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), and the distribution's 10.x line ships the module `box_sdk_gen` with a different API, so the `>=3.14.0,<4` cap is load-bearing. Port the client to `box_sdk_gen` (and drop the cap), or write down that the Box backend stays on the 3.x SDK. From 334d6760c2b3ee88e94d1b02cc0723f73001a5a9 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 05:52:39 +0800 Subject: [PATCH 10/20] Keep FileAutomation.log out of the working directory and stop printing the registry size on import --- architecture.md | 2 +- automation_file/core/action_registry.py | 4 +- automation_file/logging_config.py | 73 +++++++++++++++++++--- docs/updates/2026-09.md | 8 +++ docs/updates/README.md | 3 +- tests/test_log_location.py | 80 +++++++++++++++++++++++++ 6 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 tests/test_log_location.py diff --git a/architecture.md b/architecture.md index 5138724..5ca5563 100644 --- a/architecture.md +++ b/architecture.md @@ -26,7 +26,7 @@ the CLI, over loopback TCP or HTTP servers, as MCP tools, or from the PySide6 GU | `automation_file/project/` | `ProjectBuilder`, `create_project_dir` | | `automation_file/ui/` | PySide6 GUI: `launcher.launch_ui`, `main_window.MainWindow`, `worker.ActionWorker`, `log_widget.LogPanel`, `tabs/` (backend panels are grouped under `TransferTab`) | | `automation_file/utils/` | File discovery, fast find, grep, duplicate finder, backup rotation | -| `automation_file/exceptions.py`, `logging_config.py` | `FileAutomationException` hierarchy; `file_automation_logger` | +| `automation_file/exceptions.py`, `logging_config.py` | `FileAutomationException` hierarchy; `file_automation_logger` (INFO+ to stderr, DEBUG+ to `$FILE_AUTOMATION_LOG_FILE` or `~/.automation_file/logs/FileAutomation.log`, opened on first use) | | `stable.toml`, `dev.toml` | Packaging for `automation_file` and `automation_file_dev`. No `pyproject.toml` is committed; CI and publish copy one of these TOMLs into place | | `main_ui.py` | Development shortcut for `launch_ui()` | | `tests/`, `docs/`, `examples/mcp/` | pytest suite (fixtures in `tests/conftest.py`); Sphinx docs; MCP host configuration example | diff --git a/automation_file/core/action_registry.py b/automation_file/core/action_registry.py index 2cd75e1..c474890 100644 --- a/automation_file/core/action_registry.py +++ b/automation_file/core/action_registry.py @@ -271,7 +271,9 @@ def build_default_registry() -> ActionRegistry: _register_progress_ops(registry) _register_notify_ops(registry) _load_plugins(registry) - file_automation_logger.info( + # DEBUG, not INFO: this runs at import, and INFO is mirrored to stderr, so every import -- + # `python -m automation_file --help` included -- printed it. + file_automation_logger.debug( "action_registry: built default registry with %d commands", len(registry) ) return registry diff --git a/automation_file/logging_config.py b/automation_file/logging_config.py index 2b30e0a..8c168e0 100644 --- a/automation_file/logging_config.py +++ b/automation_file/logging_config.py @@ -1,20 +1,79 @@ """Module-level logger for automation_file. -A single :data:`file_automation_logger` is exposed. It writes to -``FileAutomation.log`` in append mode and mirrors every record to stderr via a -custom handler. The handler list is rebuilt only once, even if the module is -reloaded, so tests can import this safely. +A single :data:`file_automation_logger` is exposed. It mirrors INFO+ to stderr +and writes DEBUG+ to ``~/.automation_file/logs/FileAutomation.log`` unless +``FILE_AUTOMATION_LOG_FILE`` names another path (a relative one resolves against +the cwd at import time; ``os.devnull`` turns the file off). The handler list is +rebuilt only once, even if the module is reloaded, so tests can import this safely. + +The file used to be ``FileAutomation.log`` in the working directory, opened at +import, so every process that imported the package (PyBreeze, TestPioneer, test +runs) left one wherever it started. It is now opened on the first record, so +importing writes nothing; every process on the account appends to it with its +process id on each line, and it is rotated only when a process opens it, since +Windows cannot rename a file another process holds open. """ from __future__ import annotations import logging +import os import sys +import warnings +from logging.handlers import RotatingFileHandler +from pathlib import Path -_LOG_FORMAT = "%(asctime)s | %(name)s | %(levelname)s | %(message)s" -_LOG_FILENAME = "FileAutomation.log" +_LOG_FORMAT = "%(asctime)s | %(process)d | %(name)s | %(levelname)s | %(message)s" _LOGGER_NAME = "automation_file" +#: Environment variable that overrides where the log file is written. +LOG_FILE_ENV = "FILE_AUTOMATION_LOG_FILE" + +#: A file past this size is moved to ``.1`` when a process opens it. +ROTATE_AT_BYTES = 10 * 1024 * 1024 + + +def default_log_file() -> Path: + """Return the log file path: ``$FILE_AUTOMATION_LOG_FILE``, else the home-directory default.""" + configured = os.environ.get(LOG_FILE_ENV, "").strip() + if configured: + return Path(configured).expanduser() + return Path.home() / ".automation_file" / "logs" / "FileAutomation.log" + + +def _rotate_if_large(path: Path, limit: int) -> None: + """Move ``path`` to ``.1`` past ``limit`` bytes; best effort while another process holds it.""" + try: + if limit <= 0 or not path.is_file() or path.stat().st_size <= limit: + return + os.replace(path, path.with_name(path.name + ".1")) + except OSError: + return + + +class FileAutomationFileHandler(RotatingFileHandler): + """Append-mode UTF-8 file handler; a file that cannot be opened becomes ``os.devnull`` with one warning.""" + + def __init__(self, filename: str, delay: bool = True) -> None: + super().__init__( + filename=filename, mode="a", encoding="utf-8", errors="backslashreplace", delay=delay + ) + + def _open(self): + path = Path(self.baseFilename) + try: + path.parent.mkdir(parents=True, exist_ok=True) + _rotate_if_large(path, ROTATE_AT_BYTES) + return super()._open() + except OSError as error: + warnings.warn( + f"FileAutomation log file {path} unavailable, file logging off: {error!r}", + RuntimeWarning, + stacklevel=2, + ) + # The handler owns this stream and closes it in close(). + return open(os.devnull, self.mode, encoding=self.encoding, errors=self.errors) # pylint: disable=consider-using-with + class _StderrHandler(logging.Handler): """Mirror log records to stderr so scripts see progress without enabling root.""" @@ -35,7 +94,7 @@ def _build_logger() -> logging.Logger: formatter = logging.Formatter(_LOG_FORMAT) - file_handler = logging.FileHandler(filename=_LOG_FILENAME, mode="a", encoding="utf-8") + file_handler = FileAutomationFileHandler(str(default_log_file())) file_handler.setFormatter(formatter) file_handler.setLevel(logging.DEBUG) logger.addHandler(file_handler) diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 8b2ef0b..e61045c 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -54,3 +54,11 @@ Index and query commands: [README.md](README.md). New entries go at the end. - `.codacy_tmp/` was already ignored by `ce13593` (#1); the item just had not been removed. - **Files**: `CLAUDE.md`, `progress.md`. - **Open items**: `progress.md` #5 keeps only the port clash (9944 is also MailThunder's socket-server default). + +## U-20260923-03 · 2026-09-23 · FileAutomation.log moves out of the working directory · #done #logging + +- **What** (workspace item X-6): `logging_config.py` opened `FileAutomation.log` in the working directory at import, so every process that imported the package — PyBreeze, TestPioneer, test runs — left one wherever it started (`D:\Codes`, JEditor's and WebRunner's checkouts had collected them). It now follows the workspace scheme: `$FILE_AUTOMATION_LOG_FILE` or `~/.automation_file/logs/FileAutomation.log`, opened on the first record, append mode with the process id on each line, rotation to `.1` past 10 MiB when a process opens it, UTF-8 with `backslashreplace`, and `os.devnull` plus one warning when the path cannot be opened. +- **Also**: the "built default registry with N commands" message runs at import and was INFO, which the stderr handler mirrors, so every import — `python -m automation_file --help` included — printed it. It is DEBUG now (still in the file, no longer on stderr). +- **Tests**: `tests/test_log_location.py` (5): home default, environment override, loading the module alone leaves the cwd empty, the first record creates the directory and later handlers append (CJK, non-cp950, lone surrogate), an unopenable path only warns. Suite: 757 passed, 8 skipped; ruff check and format clean, mypy clean on the module. +- **Files**: `automation_file/logging_config.py`, `automation_file/core/action_registry.py`, `tests/test_log_location.py`, `architecture.md` §2. +- **Open items**: none here. diff --git a/docs/updates/README.md b/docs/updates/README.md index 3e9e37e..d3b6b68 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-03 | 2026-09-23 | FileAutomation.log moves out of the working directory | #done #logging | [2026-09](2026-09.md) | | U-20260923-02 | 2026-09-23 | CLAUDE.md matches the code again | #done #docs | [2026-09](2026-09.md) | | U-20260923-01 | 2026-09-23 | Dependency floors raised; Dependabot on dev; boxsdk 10 refused | #done #deps #ci | [2026-09](2026-09.md) | | U-20260922-05 | 2026-09-22 | Contract test for the legacy CLI flags | #done #tests | [2026-09](2026-09.md) | @@ -70,4 +71,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 7 | +| [2026-09.md](2026-09.md) | 2026-09 | 8 | diff --git a/tests/test_log_location.py b/tests/test_log_location.py new file mode 100644 index 0000000..6eb299f --- /dev/null +++ b/tests/test_log_location.py @@ -0,0 +1,80 @@ +"""Where FileAutomation's log goes, and that importing the package writes nothing (workspace item X-6).""" + +import logging +import os +import subprocess # nosec B404 - the import is exercised in a fresh interpreter +import sys +import warnings +from pathlib import Path + +from automation_file import logging_config as loggin_instance +from automation_file.logging_config import LOG_FILE_ENV, FileAutomationFileHandler, default_log_file + +MODULE_FILE = Path(loggin_instance.__file__) + +# Load just this module in a fresh interpreter (skipping the package __init__), then list +# the working directory. +_IMPORT_ONLY = ( + "import importlib.util, os, sys\n" + "spec = importlib.util.spec_from_file_location('probe', sys.argv[1])\n" + "spec.loader.exec_module(importlib.util.module_from_spec(spec))\n" + "print(sorted(os.listdir('.')))\n" +) + + +def _log(handler: logging.Handler, message: str) -> None: + log = logging.getLogger(f"test_log_location.{id(handler)}") + log.propagate = False + log.setLevel(logging.DEBUG) + log.addHandler(handler) + try: + log.warning(message) + finally: + log.removeHandler(handler) + handler.close() + + +def test_default_is_under_the_home_directory(monkeypatch, tmp_path): + monkeypatch.delenv(LOG_FILE_ENV, raising=False) + monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path)) + assert default_log_file() == tmp_path / ".automation_file" / "logs" / "FileAutomation.log" + + +def test_environment_variable_overrides_the_location(monkeypatch, tmp_path): + monkeypatch.setenv(LOG_FILE_ENV, str(tmp_path / "custom.log")) + assert default_log_file() == tmp_path / "custom.log" + + +def test_importing_writes_no_file(tmp_path): + target = tmp_path / "home" / "FileAutomation.log" + env = {key: value for key, value in os.environ.items() if key != LOG_FILE_ENV} + env[LOG_FILE_ENV] = str(target) + result = subprocess.run( # nosec B603 - fixed interpreter, test-controlled arguments + [sys.executable, "-c", _IMPORT_ONLY, str(MODULE_FILE)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "[]" + assert not target.parent.exists() + + +def test_first_record_creates_the_directory_and_later_handlers_append(tmp_path): + target = tmp_path / "nested" / "FileAutomation.log" + _log(FileAutomationFileHandler(str(target)), "first") + _log(FileAutomationFileHandler(str(target)), "second 中文 ⠐ \udcff") + text = target.read_text(encoding="utf-8") + assert "first" in text and "second 中文 ⠐" in text and "\\udcff" in text + + +def test_an_unopenable_path_only_warns(tmp_path): + blocker = tmp_path / "a_file" + blocker.write_text("x", encoding="utf-8") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _log(FileAutomationFileHandler(str(blocker / "FileAutomation.log")), "goes nowhere") + assert any(issubclass(item.category, RuntimeWarning) for item in caught) From 25b2e04ef165c3e6be5ed8200a530bd68f531505 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 07:01:17 +0800 Subject: [PATCH 11/20] Record MailThunder's new default port in the cross-project notes --- architecture.md | 3 ++- docs/updates/2026-09.md | 6 ++++++ docs/updates/README.md | 3 ++- progress.md | 1 - 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/architecture.md b/architecture.md index 5ca5563..038ec3f 100644 --- a/architecture.md +++ b/architecture.md @@ -120,7 +120,8 @@ ActionExecutor() → build_default_registry(): local + http + utils + drive comm (`test_pioneer/executor/file/file_processing.py`). Its `parallel_run` does not spawn this package. - **Names inherited from AutoControl**: the TCP starter is still called `start_autocontrol_socket_server` and the action-dict key is `auto_control`; MailThunder uses the same name and key. MailThunder's - socket-server default port (9944) equals this package's HTTP-server default. + socket-server default is 9942, so it runs next to this package's servers (TCP 9943, HTTP 9944, + metrics 9945) on their defaults. - **Wire format**: TCP replies end with the same `Return_Data_Over_JE` terminator as the sibling servers. - **Builtins policy**: the default registry contains no Python builtins; only `PackageLoader` can add them. In the siblings, APITestka uses an explicit allowlist, LoadDensity a `_UNSAFE_BUILTINS` diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index e61045c..a22a82d 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -62,3 +62,9 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Tests**: `tests/test_log_location.py` (5): home default, environment override, loading the module alone leaves the cwd empty, the first record creates the directory and later handlers append (CJK, non-cp950, lone surrogate), an unopenable path only warns. Suite: 757 passed, 8 skipped; ruff check and format clean, mypy clean on the module. - **Files**: `automation_file/logging_config.py`, `automation_file/core/action_registry.py`, `tests/test_log_location.py`, `architecture.md` §2. - **Open items**: none here. + +## U-20260923-04 · 2026-09-23 · Port clash with MailThunder resolved on MailThunder's side · #done #docs + +- **What** (`progress.md` #5): MailThunder's socket-server default moved from 9944 to 9942 (MailThunder U-20260923-06). This package's defaults are unchanged: TCP 9943, HTTP 9944, metrics 9945. Keeping them avoided changing the port in every README, doc page, CLI default and GUI spin box here. +- **Files**: `architecture.md` (§6 records the new MailThunder default), `progress.md`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index d3b6b68..b437ce8 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-04 | 2026-09-23 | Port clash with MailThunder resolved on MailThunder's side | #done #docs | [2026-09](2026-09.md) | | U-20260923-03 | 2026-09-23 | FileAutomation.log moves out of the working directory | #done #logging | [2026-09](2026-09.md) | | U-20260923-02 | 2026-09-23 | CLAUDE.md matches the code again | #done #docs | [2026-09](2026-09.md) | | U-20260923-01 | 2026-09-23 | Dependency floors raised; Dependabot on dev; boxsdk 10 refused | #done #deps #ci | [2026-09](2026-09.md) | @@ -71,4 +72,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 8 | +| [2026-09.md](2026-09.md) | 2026-09 | 9 | diff --git a/progress.md b/progress.md index 1d58230..5c04725 100644 --- a/progress.md +++ b/progress.md @@ -7,5 +7,4 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- ## Open - **#3** `origin/release/bump-v0.0.32` is still unmerged (workspace X-16). The five dependabot branches are settled: four floors applied on `dev` and boxsdk refused, see `docs/updates` U-20260923-01. -- **#5** The HTTP action server's default port 9944 is also MailThunder's socket-server default, so the two cannot run side by side on their defaults. Pick a free default for one of them (and note it in both `architecture.md` §6). - **#6** The Box backend (`automation_file/remote/box/client.py`) uses the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), and the distribution's 10.x line ships the module `box_sdk_gen` with a different API, so the `>=3.14.0,<4` cap is load-bearing. Port the client to `box_sdk_gen` (and drop the cap), or write down that the Box backend stays on the 3.x SDK. From 2d96aa4f44245a582806ef60c7dc80a7d9f110c1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 07:39:14 +0800 Subject: [PATCH 12/20] Move the Box backend to box_sdk_gen and require boxsdk 10 The legacy boxsdk.Client API is no longer developed and kept the dependency capped below 4. Uploads, downloads, listing and deletes now go through the box_sdk_gen managers; the public functions are unchanged. --- automation_file/remote/box/client.py | 47 ++++++----- automation_file/remote/box/delete_ops.py | 4 +- automation_file/remote/box/download_ops.py | 2 +- automation_file/remote/box/list_ops.py | 12 ++- automation_file/remote/box/upload_ops.py | 19 +++-- dev.toml | 2 +- docs/updates/2026-09.md | 18 ++++ docs/updates/README.md | 3 +- progress.md | 1 - requirements.txt | 2 +- stable.toml | 2 +- tests/test_box_ops.py | 97 +++++++++++++--------- 12 files changed, 129 insertions(+), 80 deletions(-) diff --git a/automation_file/remote/box/client.py b/automation_file/remote/box/client.py index ea63b36..2478af2 100644 --- a/automation_file/remote/box/client.py +++ b/automation_file/remote/box/client.py @@ -1,10 +1,11 @@ -"""Box client (Singleton Facade) backed by the ``boxsdk`` library. +"""Box client (Singleton Facade) backed by ``box_sdk_gen``. -Box's OAuth2 flow is authorization-code based (not device-code), so the -caller is expected to obtain an access token via their app registration -and hand it in to :meth:`later_init`. Matches the Dropbox backend's -contract — automation workflows typically receive the token from a -secrets manager rather than prompting interactively. +``box_sdk_gen`` is the module shipped by the ``boxsdk`` distribution from 10.0 on; the legacy +``boxsdk.Client`` API it replaced is no longer developed. Box's OAuth2 flow is authorization-code +based (not device-code), so the caller is expected to obtain an access token via their app +registration and hand it in to :meth:`later_init`. Matches the Dropbox backend's contract — +automation workflows typically receive the token from a secrets manager rather than prompting +interactively. """ from __future__ import annotations @@ -15,18 +16,19 @@ from automation_file.logging_config import file_automation_logger -def _import_boxsdk() -> Any: +def import_box_sdk_gen() -> Any: + """Import ``box_sdk_gen``, raising :class:`BoxException` when it is missing.""" try: - import boxsdk + import box_sdk_gen except ImportError as error: raise BoxException( - "boxsdk import failed — reinstall `automation_file` to restore the Box backend" + "box_sdk_gen import failed — install `boxsdk>=10` to restore the Box backend" ) from error - return boxsdk + return box_sdk_gen class BoxClient: - """Lazy wrapper around :class:`boxsdk.Client`.""" + """Lazy wrapper around :class:`box_sdk_gen.BoxClient`.""" def __init__(self) -> None: self.client: Any = None @@ -38,25 +40,26 @@ def later_init( client_id: str = "", client_secret: str = "", ) -> Any: - """Build a :class:`boxsdk.Client` from an OAuth2 access token. + """Build a :class:`box_sdk_gen.BoxClient` from an OAuth2 access token. - ``client_id`` and ``client_secret`` are only required if the caller - wants to let boxsdk refresh the token — most automation callers - already refresh externally, so both default to empty. + ``client_id`` and ``client_secret`` are optional; when given they are passed to the + developer-token auth so the token can be revoked through the SDK. Refreshing the token + stays the caller's job, as most automation callers already refresh it externally. """ if not isinstance(access_token, str) or not access_token: raise BoxException("access_token must be a non-empty string") - boxsdk = _import_boxsdk() - oauth = boxsdk.OAuth2( - client_id=client_id, - client_secret=client_secret, - access_token=access_token, - ) - self.client = boxsdk.Client(oauth) + sdk = import_box_sdk_gen() + config = None + if client_id or client_secret: + config = sdk.DeveloperTokenConfig(client_id=client_id or None, + client_secret=client_secret or None) + auth = sdk.BoxDeveloperTokenAuth(token=access_token, config=config) + self.client = sdk.BoxClient(auth=auth) file_automation_logger.info("BoxClient: client ready") return self.client def require_client(self) -> Any: + """Return the initialised SDK client, or raise :class:`BoxException` before ``later_init``.""" if self.client is None: raise BoxException("BoxClient not initialised; call later_init() first") return self.client diff --git a/automation_file/remote/box/delete_ops.py b/automation_file/remote/box/delete_ops.py index 09f2259..b3d6350 100644 --- a/automation_file/remote/box/delete_ops.py +++ b/automation_file/remote/box/delete_ops.py @@ -11,7 +11,7 @@ def box_delete_file(file_id: str) -> bool: """Delete a Box file by id.""" client = box_instance.require_client() try: - client.file(file_id=file_id).delete() + client.files.delete_file_by_id(file_id) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_delete_file failed: {error}") from error file_automation_logger.info("box_delete_file: %s", file_id) @@ -22,7 +22,7 @@ def box_delete_folder(folder_id: str, recursive: bool = False) -> bool: """Delete a Box folder by id (optionally recursive).""" client = box_instance.require_client() try: - client.folder(folder_id=folder_id).delete(recursive=recursive) + client.folders.delete_folder_by_id(folder_id, recursive=recursive) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_delete_folder failed: {error}") from error file_automation_logger.info("box_delete_folder: %s (recursive=%s)", folder_id, recursive) diff --git a/automation_file/remote/box/download_ops.py b/automation_file/remote/box/download_ops.py index d055128..313bdb7 100644 --- a/automation_file/remote/box/download_ops.py +++ b/automation_file/remote/box/download_ops.py @@ -16,7 +16,7 @@ def box_download_file(file_id: str, target_path: str) -> bool: target.parent.mkdir(parents=True, exist_ok=True) try: with open(target, "wb") as writer: - client.file(file_id=file_id).download_to(writer) + client.downloads.download_file_to_output_stream(file_id, writer) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_download_file failed: {error}") from error file_automation_logger.info("box_download_file: %s -> %s", file_id, target) diff --git a/automation_file/remote/box/list_ops.py b/automation_file/remote/box/list_ops.py index 58a5b96..f4b12a8 100644 --- a/automation_file/remote/box/list_ops.py +++ b/automation_file/remote/box/list_ops.py @@ -9,6 +9,11 @@ from automation_file.remote.box.client import box_instance +def _type_name(item_type: Any) -> str: + """``"file"``, ``"folder"`` or ``"web_link"``: the SDK reports the type as an enum.""" + return str(getattr(item_type, "value", item_type)) + + def box_list_folder(folder_id: str = "0", limit: int = 100) -> list[dict[str, Any]]: """List entries in a Box folder; return basic metadata per entry. @@ -19,13 +24,12 @@ def box_list_folder(folder_id: str = "0", limit: int = 100) -> list[dict[str, An """ client = box_instance.require_client() try: - folder = client.folder(folder_id=folder_id) - items = folder.get_items(limit=limit) + items = client.folders.get_folder_items(folder_id, limit=limit).entries or [] entries = [ { "id": str(getattr(item, "id", "")), - "name": getattr(item, "name", ""), - "type": getattr(item, "type", "file"), + "name": getattr(item, "name", "") or "", + "type": _type_name(getattr(item, "type", "file")), } for item in items ] diff --git a/automation_file/remote/box/upload_ops.py b/automation_file/remote/box/upload_ops.py index 2e2a0d9..6d37da1 100644 --- a/automation_file/remote/box/upload_ops.py +++ b/automation_file/remote/box/upload_ops.py @@ -7,7 +7,7 @@ from automation_file.exceptions import BoxException, FileNotExistsException from automation_file.logging_config import file_automation_logger from automation_file.remote._upload_tree import walk_and_upload -from automation_file.remote.box.client import box_instance +from automation_file.remote.box.client import box_instance, import_box_sdk_gen def box_upload_file(file_path: str, parent_folder_id: str = "0", name: str = "") -> str: @@ -22,19 +22,20 @@ def box_upload_file(file_path: str, parent_folder_id: str = "0", name: str = "") raise FileNotExistsException(str(local)) client = box_instance.require_client() target_name = name or local.name + sdk = import_box_sdk_gen() + attributes = sdk.UploadFileAttributes( + name=target_name, parent=sdk.UploadFileAttributesParentField(id=parent_folder_id) + ) try: - folder = client.folder(folder_id=parent_folder_id) - new_file = folder.upload(file_path=str(local), file_name=target_name) + with open(local, "rb") as stream: + uploaded = client.uploads.upload_file(attributes, stream) + new_id = str(uploaded.entries[0].id) except Exception as error: # pylint: disable=broad-except raise BoxException(f"box_upload_file failed: {error}") from error file_automation_logger.info( - "box_upload_file: %s -> %s/%s (id=%s)", - local, - parent_folder_id, - target_name, - getattr(new_file, "id", "?"), + "box_upload_file: %s -> %s/%s (id=%s)", local, parent_folder_id, target_name, new_id ) - return str(getattr(new_file, "id", "")) + return new_id def box_upload_dir(dir_path: str, parent_folder_id: str = "0") -> list[str]: diff --git a/dev.toml b/dev.toml index 903b0b2..56a7ca3 100644 --- a/dev.toml +++ b/dev.toml @@ -33,7 +33,7 @@ dependencies = [ "opentelemetry-api>=1.41.1", "opentelemetry-sdk>=1.42.1", "msal>=1.37.0", - "boxsdk>=3.14.0,<4", + "boxsdk>=10.0.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index a22a82d..0bff959 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -68,3 +68,21 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What** (`progress.md` #5): MailThunder's socket-server default moved from 9944 to 9942 (MailThunder U-20260923-06). This package's defaults are unchanged: TCP 9943, HTTP 9944, metrics 9945. Keeping them avoided changing the port in every README, doc page, CLI default and GUI spin box here. - **Files**: `architecture.md` (§6 records the new MailThunder default), `progress.md`. - **Open items**: none. + +## U-20260923-05 · 2026-09-23 · Box backend moves to box_sdk_gen (boxsdk 10) · #done #deps #box + +- **What** (`progress.md` #6): the Box backend used the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), which kept the dependency capped at `<4`. Box consolidated its SDKs: from 10.0 the `boxsdk` distribution ships only `box_sdk_gen`, and Box recommends migrating to 10.x, with support for the consolidated SDKs continuing into 2027. The backend now uses `box_sdk_gen`: + - `BoxClient.later_init` builds `BoxDeveloperTokenAuth` (with `DeveloperTokenConfig` when a client id or secret is given) and `box_sdk_gen.BoxClient`. The signature is unchanged. + - upload → `client.uploads.upload_file(UploadFileAttributes(name, parent=…), stream)`, returning `entries[0].id` + - download → `client.downloads.download_file_to_output_stream` + - list → `client.folders.get_folder_items(folder_id, limit=…)`. The item type is an enum there, so `box_list_folder` still returns `"file"`, `"folder"` or `"web_link"` strings. + - delete → `client.files.delete_file_by_id` and `client.folders.delete_folder_by_id(…, recursive=…)` +- **Dependency**: `boxsdk>=10.0.0,<11` in `requirements.txt`, `dev.toml` and `stable.toml`. The 3.x cap is gone. +- **How it was checked**: boxsdk 10.15.0 was installed in a scratch venv and each signature used above was read from the installed package. The FileAutomation venv now has boxsdk 10.7.0. +- **Tests**: + - `tests/test_box_ops.py`: the fakes now mirror the SDK's managers (`uploads`, `downloads`, `folders`, `files`), and the tests check the arguments each call receives (name, parent id, content, recursive flag, the ids deleted). + - New: `later_init` builds a real `box_sdk_gen.BoxClient`. + - Full suite: 759 passed, 8 skipped. + - There are still no calls against a live Box account. +- **Files**: `automation_file/remote/box/{client,upload_ops,download_ops,list_ops,delete_ops}.py`, `tests/test_box_ops.py`, `requirements.txt`, `dev.toml`, `stable.toml`, `progress.md`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index b437ce8..6e73496 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-05 | 2026-09-23 | Box backend moves to box_sdk_gen (boxsdk 10) | #done #deps #box | [2026-09](2026-09.md) | | U-20260923-04 | 2026-09-23 | Port clash with MailThunder resolved on MailThunder's side | #done #docs | [2026-09](2026-09.md) | | U-20260923-03 | 2026-09-23 | FileAutomation.log moves out of the working directory | #done #logging | [2026-09](2026-09.md) | | U-20260923-02 | 2026-09-23 | CLAUDE.md matches the code again | #done #docs | [2026-09](2026-09.md) | @@ -72,4 +73,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 9 | +| [2026-09.md](2026-09.md) | 2026-09 | 10 | diff --git a/progress.md b/progress.md index 5c04725..9596a7e 100644 --- a/progress.md +++ b/progress.md @@ -7,4 +7,3 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- ## Open - **#3** `origin/release/bump-v0.0.32` is still unmerged (workspace X-16). The five dependabot branches are settled: four floors applied on `dev` and boxsdk refused, see `docs/updates` U-20260923-01. -- **#6** The Box backend (`automation_file/remote/box/client.py`) uses the legacy `boxsdk` 3.x API (`boxsdk.OAuth2`, `boxsdk.Client`), and the distribution's 10.x line ships the module `box_sdk_gen` with a different API, so the `>=3.14.0,<4` cap is load-bearing. Port the client to `box_sdk_gen` (and drop the cap), or write down that the Box backend stays on the 3.x SDK. diff --git a/requirements.txt b/requirements.txt index 76e9db9..bf36a4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,5 @@ pyarrow>=15.0.0 opentelemetry-api>=1.41.1 opentelemetry-sdk>=1.41.1 msal>=1.36.0 -boxsdk>=3.14.0,<4 +boxsdk>=10.0.0,<11 tomli; python_version<"3.11" \ No newline at end of file diff --git a/stable.toml b/stable.toml index c2c4820..bcb6a74 100644 --- a/stable.toml +++ b/stable.toml @@ -33,7 +33,7 @@ dependencies = [ "opentelemetry-api>=1.41.1", "opentelemetry-sdk>=1.42.1", "msal>=1.37.0", - "boxsdk>=3.14.0,<4", + "boxsdk>=10.0.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ diff --git a/tests/test_box_ops.py b/tests/test_box_ops.py index ab88d07..fd972cc 100644 --- a/tests/test_box_ops.py +++ b/tests/test_box_ops.py @@ -2,15 +2,17 @@ Live Box endpoints are outside CI; these tests verify registry wiring, the Client singleton's guard clauses, and the error-path wrapping that -converts ``boxsdk`` failures into :class:`BoxException`. +converts ``box_sdk_gen`` failures into :class:`BoxException`. """ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest +from box_sdk_gen.schemas.file_base import FileBaseTypeField from automation_file import ( BoxClient, @@ -25,52 +27,57 @@ class _FakeItem: - def __init__(self, item_id: str, name: str, item_type: str = "file") -> None: + def __init__(self, item_id: str, name: str, item_type: Any) -> None: self.id = item_id self.name = name self.type = item_type -class _FakeFile: - def __init__(self, file_id: str) -> None: - self.id = file_id +class _Uploads: + def __init__(self) -> None: + self.calls: list[tuple[str, str, bytes]] = [] - def download_to(self, writer: Any) -> None: - writer.write(b"contents") + def upload_file(self, attributes: Any, file: Any) -> Any: + self.calls.append((attributes.name, attributes.parent.id, file.read())) + return SimpleNamespace(entries=[SimpleNamespace(id="new-id")]) - def delete(self) -> None: - return None +class _Downloads: + def download_file_to_output_stream(self, file_id: str, output_stream: Any) -> None: + output_stream.write(f"contents of {file_id}".encode()) -class _FakeFolder: - def __init__(self, folder_id: str) -> None: - self.id = folder_id - self._uploads: list[tuple[str, str]] = [] - def upload(self, file_path: str, file_name: str) -> _FakeFile: - self._uploads.append((file_path, file_name)) - return _FakeFile("new-id") +class _Folders: + def __init__(self) -> None: + self.deleted: list[tuple[str, bool]] = [] - def get_items(self, limit: int = 100) -> list[_FakeItem]: - del limit - return [_FakeItem("1", "a.txt"), _FakeItem("2", "subdir", "folder")] + def get_folder_items(self, folder_id: str, *, limit: int | None = None) -> Any: + del folder_id, limit + return SimpleNamespace(entries=[ + _FakeItem("1", "a.txt", FileBaseTypeField.FILE), + _FakeItem("2", "subdir", "folder"), + ]) - def delete(self, recursive: bool = False) -> None: - del recursive + def delete_folder_by_id(self, folder_id: str, *, recursive: bool | None = None) -> None: + self.deleted.append((folder_id, bool(recursive))) -class _FakeBoxClient: +class _Files: def __init__(self) -> None: - self._files: dict[str, _FakeFile] = {} - self._folders: dict[str, _FakeFolder] = {} + self.deleted: list[str] = [] + + def delete_file_by_id(self, file_id: str) -> None: + self.deleted.append(file_id) - def file(self, file_id: str) -> _FakeFile: - self._files.setdefault(file_id, _FakeFile(file_id)) - return self._files[file_id] - def folder(self, folder_id: str) -> _FakeFolder: - self._folders.setdefault(folder_id, _FakeFolder(folder_id)) - return self._folders[folder_id] +class _FakeBoxClient: + """Mirrors the managers of :class:`box_sdk_gen.BoxClient` that the backend calls.""" + + def __init__(self) -> None: + self.uploads = _Uploads() + self.downloads = _Downloads() + self.folders = _Folders() + self.files = _Files() @pytest.fixture(name="fake_box") @@ -129,16 +136,15 @@ def test_upload_dir_uploads_each_file(tmp_path: Path, fake_box: _FakeBoxClient) (tmp_path / "sub" / "b.txt").write_text("b", encoding="utf-8") uploaded_keys = upload_ops.box_upload_dir(str(tmp_path)) assert sorted(uploaded_keys) == ["a.txt", "sub/b.txt"] - folder = fake_box.folder("0") - flat_names = sorted(name for _, name in folder._uploads) # pylint: disable=protected-access - assert flat_names == ["a.txt", "sub/b.txt"] + assert sorted((name, parent) for name, parent, _ in fake_box.uploads.calls) == [ + ("a.txt", "0"), ("sub/b.txt", "0")] def test_download_writes_target(tmp_path: Path, fake_box: _FakeBoxClient) -> None: del fake_box target = tmp_path / "out" / "f.txt" assert download_ops.box_download_file("42", str(target)) is True - assert target.read_bytes() == b"contents" + assert target.read_bytes() == b"contents of 42" def test_list_folder_returns_entries(fake_box: _FakeBoxClient) -> None: @@ -151,13 +157,13 @@ def test_list_folder_returns_entries(fake_box: _FakeBoxClient) -> None: def test_delete_file_uses_client(fake_box: _FakeBoxClient) -> None: - del fake_box assert delete_ops.box_delete_file("7") is True + assert fake_box.files.deleted == ["7"] def test_delete_folder_uses_client(fake_box: _FakeBoxClient) -> None: - del fake_box assert delete_ops.box_delete_folder("7", recursive=True) is True + assert fake_box.folders.deleted == [("7", True)] def test_errors_in_sdk_surface_as_box_exception( @@ -166,6 +172,23 @@ def test_errors_in_sdk_surface_as_box_exception( def blow(*_a: Any, **_k: Any) -> None: raise RuntimeError("simulated SDK error") - monkeypatch.setattr(fake_box, "folder", blow) + monkeypatch.setattr(fake_box.folders, "get_folder_items", blow) with pytest.raises(BoxException): list_ops.box_list_folder() + + +def test_upload_file_sends_name_parent_and_content(tmp_path: Path, fake_box: _FakeBoxClient) -> None: + src = tmp_path / "report.txt" + src.write_bytes(b"payload") + upload_ops.box_upload_file(str(src), parent_folder_id="99", name="renamed.txt") + assert fake_box.uploads.calls == [("renamed.txt", "99", b"payload")] + + +def test_later_init_builds_a_real_sdk_client() -> None: + import box_sdk_gen + + client = BoxClient() + built = client.later_init("token-value", client_id="id", client_secret="secret") + assert isinstance(built, box_sdk_gen.BoxClient) + assert client.require_client() is built + assert hasattr(built, "uploads") and hasattr(built, "folders") From 6b16d087a1496fb3449be0aa4b5b3ddb154e01cd Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 07:42:15 +0800 Subject: [PATCH 13/20] Require cryptography 50 and msal 1.39 cryptography 50.0.0 fixes PYSEC-2026-3552; msal before 1.39 caps cryptography below 49, so both floors move together. --- dev.toml | 4 ++-- docs/updates/2026-09.md | 10 ++++++++++ docs/updates/README.md | 3 ++- requirements.txt | 2 +- stable.toml | 4 ++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/dev.toml b/dev.toml index 56a7ca3..9c29906 100644 --- a/dev.toml +++ b/dev.toml @@ -25,14 +25,14 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=49.0.0", + "cryptography>=50.0.0", "prometheus_client>=0.25.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", "pyarrow>=24.0.0", "opentelemetry-api>=1.41.1", "opentelemetry-sdk>=1.42.1", - "msal>=1.37.0", + "msal>=1.39.0", "boxsdk>=10.0.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 0bff959..dd8a54b 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -86,3 +86,13 @@ Index and query commands: [README.md](README.md). New entries go at the end. - There are still no calls against a live Box account. - **Files**: `automation_file/remote/box/{client,upload_ops,download_ops,list_ops,delete_ops}.py`, `tests/test_box_ops.py`, `requirements.txt`, `dev.toml`, `stable.toml`, `progress.md`. - **Open items**: none. + +## U-20260923-06 · 2026-09-23 · cryptography floor 50 and msal 1.39 · #done #deps #security + +- **Found by**: running `pip-audit` against the local venv. The venv had cryptography 46.0.7, and advisories PYSEC-2026-3552 and PYSEC-2026-3554 are fixed in 49.0.0 and 50.0.0. The declared floor `cryptography>=49.0.0` still allowed a 49.x without the 50.0.0 fix. +- **Change**: + - `cryptography>=50.0.0` in `dev.toml` and `stable.toml`. + - `msal>=1.39.0` there and in `requirements.txt`. msal releases before 1.39 require `cryptography<49`, so the two floors have to move together. +- **Local venv**: the flagged packages were upgraded (cryptography 50.0.1, msal 1.39.0, httplib2, idna, pyasn1, urllib3, pip), and `uv pip check` is clean. Suite: 759 passed, 8 skipped. +- **Files**: `dev.toml`, `stable.toml`, `requirements.txt`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index 6e73496..a510a1d 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-06 | 2026-09-23 | cryptography floor 50 and msal 1.39 | #done #deps #security | [2026-09](2026-09.md) | | U-20260923-05 | 2026-09-23 | Box backend moves to box_sdk_gen (boxsdk 10) | #done #deps #box | [2026-09](2026-09.md) | | U-20260923-04 | 2026-09-23 | Port clash with MailThunder resolved on MailThunder's side | #done #docs | [2026-09](2026-09.md) | | U-20260923-03 | 2026-09-23 | FileAutomation.log moves out of the working directory | #done #logging | [2026-09](2026-09.md) | @@ -73,4 +74,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 10 | +| [2026-09.md](2026-09.md) | 2026-09 | 11 | diff --git a/requirements.txt b/requirements.txt index bf36a4f..65180ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,6 @@ PyYAML>=6.0.3 pyarrow>=15.0.0 opentelemetry-api>=1.41.1 opentelemetry-sdk>=1.41.1 -msal>=1.36.0 +msal>=1.39.0 boxsdk>=10.0.0,<11 tomli; python_version<"3.11" \ No newline at end of file diff --git a/stable.toml b/stable.toml index bcb6a74..8f7a54f 100644 --- a/stable.toml +++ b/stable.toml @@ -25,14 +25,14 @@ dependencies = [ "paramiko>=3.4.0", "PySide6>=6.6.0", "watchdog>=4.0.0", - "cryptography>=49.0.0", + "cryptography>=50.0.0", "prometheus_client>=0.25.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", "pyarrow>=24.0.0", "opentelemetry-api>=1.41.1", "opentelemetry-sdk>=1.42.1", - "msal>=1.37.0", + "msal>=1.39.0", "boxsdk>=10.0.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] From 42b8ce0194325ca805b9bc9710e9e8abd59cfad1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 08:09:41 +0800 Subject: [PATCH 14/20] Record the deletion of the stale release/bump-v0.0.32 branch --- docs/updates/2026-09.md | 5 +++++ docs/updates/README.md | 3 ++- progress.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index dd8a54b..d6c3072 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -96,3 +96,8 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Local venv**: the flagged packages were upgraded (cryptography 50.0.1, msal 1.39.0, httplib2, idna, pyasn1, urllib3, pip), and `uv pip check` is clean. Suite: 759 passed, 8 skipped. - **Files**: `dev.toml`, `stable.toml`, `requirements.txt`. - **Open items**: none. + +## U-20260923-07 · 2026-09-23 · Stale release/bump-v0.0.32 branch deleted · #done #housekeeping + +- **What** (`progress.md` #3, workspace X-16): `origin/release/bump-v0.0.32` held a single bot commit, `0014231` ("Bump version to v0.0.32 [skip ci]", 2026-04-21), which changed only the version lines in `dev.toml` and `stable.toml`. `main` is at 0.0.47, so merging it would move the version backwards. It had no pull request. The branch was deleted from `origin`. If it is ever needed, restore it with `git push origin 0014231:refs/heads/release/bump-v0.0.32`. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index a510a1d..a0b31bb 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-07 | 2026-09-23 | Stale release/bump-v0.0.32 branch deleted | #done #housekeeping | [2026-09](2026-09.md) | | U-20260923-06 | 2026-09-23 | cryptography floor 50 and msal 1.39 | #done #deps #security | [2026-09](2026-09.md) | | U-20260923-05 | 2026-09-23 | Box backend moves to box_sdk_gen (boxsdk 10) | #done #deps #box | [2026-09](2026-09.md) | | U-20260923-04 | 2026-09-23 | Port clash with MailThunder resolved on MailThunder's side | #done #docs | [2026-09](2026-09.md) | @@ -74,4 +75,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 11 | +| [2026-09.md](2026-09.md) | 2026-09 | 12 | diff --git a/progress.md b/progress.md index 9596a7e..77dfd04 100644 --- a/progress.md +++ b/progress.md @@ -6,4 +6,3 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- ## Open -- **#3** `origin/release/bump-v0.0.32` is still unmerged (workspace X-16). The five dependabot branches are settled: four floors applied on `dev` and boxsdk refused, see `docs/updates` U-20260923-01. From 18748018712e95e24e318e48169860594edd3ef5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 11:21:14 +0800 Subject: [PATCH 15/20] Fix the lint job: format three files and type WebDAV request kwargs The dev lint job failed at ruff format --check since 2026-09-22 (the Box client and its test, and the legacy CLI contract test). Typing the WebDAV _request kwargs as Any also lets mypy pass with the requests stubs installed. --- automation_file/remote/box/client.py | 5 +++-- automation_file/remote/webdav/client.py | 3 ++- tests/test_box_ops.py | 18 ++++++++++++------ tests/test_legacy_cli_contract.py | 14 +++++++++++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/automation_file/remote/box/client.py b/automation_file/remote/box/client.py index 2478af2..f191a15 100644 --- a/automation_file/remote/box/client.py +++ b/automation_file/remote/box/client.py @@ -51,8 +51,9 @@ def later_init( sdk = import_box_sdk_gen() config = None if client_id or client_secret: - config = sdk.DeveloperTokenConfig(client_id=client_id or None, - client_secret=client_secret or None) + config = sdk.DeveloperTokenConfig( + client_id=client_id or None, client_secret=client_secret or None + ) auth = sdk.BoxDeveloperTokenAuth(token=access_token, config=config) self.client = sdk.BoxClient(auth=auth) file_automation_logger.info("BoxClient: client ready") diff --git a/automation_file/remote/webdav/client.py b/automation_file/remote/webdav/client.py index 08b0e8c..ab787f6 100644 --- a/automation_file/remote/webdav/client.py +++ b/automation_file/remote/webdav/client.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from pathlib import Path from types import TracebackType +from typing import Any from urllib.parse import quote, unquote, urlparse import requests @@ -91,7 +92,7 @@ def _url_for(self, remote_path: str) -> str: return self._base_url + "/" return f"{self._base_url}/{quote(remote_path, safe='/')}" - def _request(self, method: str, remote_path: str, **kwargs: object) -> requests.Response: + def _request(self, method: str, remote_path: str, **kwargs: Any) -> requests.Response: url = self._url_for(remote_path) try: response = self._session.request( diff --git a/tests/test_box_ops.py b/tests/test_box_ops.py index fd972cc..10337fb 100644 --- a/tests/test_box_ops.py +++ b/tests/test_box_ops.py @@ -53,10 +53,12 @@ def __init__(self) -> None: def get_folder_items(self, folder_id: str, *, limit: int | None = None) -> Any: del folder_id, limit - return SimpleNamespace(entries=[ - _FakeItem("1", "a.txt", FileBaseTypeField.FILE), - _FakeItem("2", "subdir", "folder"), - ]) + return SimpleNamespace( + entries=[ + _FakeItem("1", "a.txt", FileBaseTypeField.FILE), + _FakeItem("2", "subdir", "folder"), + ] + ) def delete_folder_by_id(self, folder_id: str, *, recursive: bool | None = None) -> None: self.deleted.append((folder_id, bool(recursive))) @@ -137,7 +139,9 @@ def test_upload_dir_uploads_each_file(tmp_path: Path, fake_box: _FakeBoxClient) uploaded_keys = upload_ops.box_upload_dir(str(tmp_path)) assert sorted(uploaded_keys) == ["a.txt", "sub/b.txt"] assert sorted((name, parent) for name, parent, _ in fake_box.uploads.calls) == [ - ("a.txt", "0"), ("sub/b.txt", "0")] + ("a.txt", "0"), + ("sub/b.txt", "0"), + ] def test_download_writes_target(tmp_path: Path, fake_box: _FakeBoxClient) -> None: @@ -177,7 +181,9 @@ def blow(*_a: Any, **_k: Any) -> None: list_ops.box_list_folder() -def test_upload_file_sends_name_parent_and_content(tmp_path: Path, fake_box: _FakeBoxClient) -> None: +def test_upload_file_sends_name_parent_and_content( + tmp_path: Path, fake_box: _FakeBoxClient +) -> None: src = tmp_path / "report.txt" src.write_bytes(b"payload") upload_ops.box_upload_file(str(src), parent_folder_id="99", name="renamed.txt") diff --git a/tests/test_legacy_cli_contract.py b/tests/test_legacy_cli_contract.py index 9b79067..94c3b1b 100644 --- a/tests/test_legacy_cli_contract.py +++ b/tests/test_legacy_cli_contract.py @@ -8,6 +8,7 @@ these, so this file does: renaming or removing a flag, or dropping the second JSON decode on Windows, breaks them (workspace item X-7). """ + import json import os import subprocess # nosec B404 - the CLI is exercised as a real child process @@ -30,8 +31,13 @@ def _run_cli(cwd: Path, *args: str) -> subprocess.CompletedProcess: env["PYTHONIOENCODING"] = "utf-8" return subprocess.run( # nosec B603 - fixed interpreter, test-controlled arguments [sys.executable, "-m", PACKAGE, *args], - cwd=cwd, env=env, capture_output=True, text=True, encoding="utf-8", - timeout=300, check=False, + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + timeout=300, + check=False, ) @@ -70,7 +76,9 @@ def test_execute_dir(tmp_path, flag): def test_execute_str_as_pybreeze_sends_it(tmp_path): target = tmp_path / "created.txt" - _assert_ran(_run_cli(tmp_path, "--execute_str", _pybreeze_execute_str(_actions(target))), target) + _assert_ran( + _run_cli(tmp_path, "--execute_str", _pybreeze_execute_str(_actions(target))), target + ) @pytest.mark.parametrize("flag", ["-c", "--create_project"]) From 3b0e680096a97d91a9bc7cf260abc6d32f0754f0 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 11:27:07 +0800 Subject: [PATCH 16/20] Record the dev CI repair --- docs/updates/2026-09.md | 7 +++++++ docs/updates/README.md | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index d6c3072..3b399df 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -101,3 +101,10 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **What** (`progress.md` #3, workspace X-16): `origin/release/bump-v0.0.32` held a single bot commit, `0014231` ("Bump version to v0.0.32 [skip ci]", 2026-04-21), which changed only the version lines in `dev.toml` and `stable.toml`. `main` is at 0.0.47, so merging it would move the version backwards. It had no pull request. The branch was deleted from `origin`. If it is ever needed, restore it with `git push origin 0014231:refs/heads/release/bump-v0.0.32`. - **Open items**: none. + +## U-20260923-08 · 2026-09-23 · dev CI green again: format check and mypy · #done #ci + +- **What**: every push to `dev` since 2026-09-22 failed the `lint` job at `ruff format --check`, eight runs in a row. The files were `automation_file/remote/box/client.py` and `tests/test_box_ops.py` (U-20260923-05) and `tests/test_legacy_cli_contract.py` (an earlier change). The pytest jobs never ran behind it. All three are formatted. +- **Also**: `mypy automation_file` reported 11 errors in `remote/webdav/client.py`, because `_request(**kwargs: object)` cannot satisfy the requests type stubs. CI does not install those stubs, so it would not have seen the errors. The kwargs are now typed `Any`, and mypy passes with the stubs as well. +- **Result**: run for `1874801`: lint and pytest on 3.10–3.14 all pass. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index a0b31bb..88187ce 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-08 | 2026-09-23 | dev CI green again: format check and mypy | #done #ci | [2026-09](2026-09.md) | | U-20260923-07 | 2026-09-23 | Stale release/bump-v0.0.32 branch deleted | #done #housekeeping | [2026-09](2026-09.md) | | U-20260923-06 | 2026-09-23 | cryptography floor 50 and msal 1.39 | #done #deps #security | [2026-09](2026-09.md) | | U-20260923-05 | 2026-09-23 | Box backend moves to box_sdk_gen (boxsdk 10) | #done #deps #box | [2026-09](2026-09.md) | @@ -75,4 +76,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 12 | +| [2026-09.md](2026-09.md) | 2026-09 | 13 | From c15d65a295cac21eaf7bc209ec1074e0a3a5514c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 11:31:32 +0800 Subject: [PATCH 17/20] Note MailThunder's renamed socket function and action key in section 6 --- architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/architecture.md b/architecture.md index 038ec3f..03ef11e 100644 --- a/architecture.md +++ b/architecture.md @@ -119,7 +119,7 @@ ActionExecutor() → build_default_registry(): local + http + utils + drive comm - **TestPioneer** imports `download_file` and `unzip_all` from the facade in-process (`test_pioneer/executor/file/file_processing.py`). Its `parallel_run` does not spawn this package. - **Names inherited from AutoControl**: the TCP starter is still called `start_autocontrol_socket_server` - and the action-dict key is `auto_control`; MailThunder uses the same name and key. MailThunder's + and the action-dict key is `auto_control`. MailThunder has renamed both (`start_mail_thunder_socket_server`, `mail_thunder` key) and keeps the old names as deprecated aliases. MailThunder's socket-server default is 9942, so it runs next to this package's servers (TCP 9943, HTTP 9944, metrics 9945) on their defaults. - **Wire format**: TCP replies end with the same `Return_Data_Over_JE` terminator as the sibling servers. From f6fcecffcd859febceab5b4706b33b3d44fc420a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 12:06:47 +0800 Subject: [PATCH 18/20] Raise five dependency floors from the Dependabot PRs on main prometheus_client 0.26.0, opentelemetry-api/-sdk 1.44.0, pyarrow 25.0.1 and boxsdk 10.15.0 (still below 11). requirements.txt's pyarrow floor (15.0.0) now matches the TOMLs. --- dev.toml | 10 +++++----- requirements.txt | 8 ++++---- stable.toml | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dev.toml b/dev.toml index 9c29906..8e80e96 100644 --- a/dev.toml +++ b/dev.toml @@ -26,14 +26,14 @@ dependencies = [ "PySide6>=6.6.0", "watchdog>=4.0.0", "cryptography>=50.0.0", - "prometheus_client>=0.25.0", + "prometheus_client>=0.26.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=24.0.0", - "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.42.1", + "pyarrow>=25.0.1", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", "msal>=1.39.0", - "boxsdk>=10.0.0,<11", + "boxsdk>=10.15.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ diff --git a/requirements.txt b/requirements.txt index 65180ad..c4c1b88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,9 @@ tqdm watchdog defusedxml>=0.7.1 PyYAML>=6.0.3 -pyarrow>=15.0.0 -opentelemetry-api>=1.41.1 -opentelemetry-sdk>=1.41.1 +pyarrow>=25.0.1 +opentelemetry-api>=1.44.0 +opentelemetry-sdk>=1.44.0 msal>=1.39.0 -boxsdk>=10.0.0,<11 +boxsdk>=10.15.0,<11 tomli; python_version<"3.11" \ No newline at end of file diff --git a/stable.toml b/stable.toml index 8f7a54f..02f55a0 100644 --- a/stable.toml +++ b/stable.toml @@ -26,14 +26,14 @@ dependencies = [ "PySide6>=6.6.0", "watchdog>=4.0.0", "cryptography>=50.0.0", - "prometheus_client>=0.25.0", + "prometheus_client>=0.26.0", "defusedxml>=0.7.1", "PyYAML>=6.0.3", - "pyarrow>=24.0.0", - "opentelemetry-api>=1.41.1", - "opentelemetry-sdk>=1.42.1", + "pyarrow>=25.0.1", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", "msal>=1.39.0", - "boxsdk>=10.0.0,<11", + "boxsdk>=10.15.0,<11", "tomli>=2.0.1; python_version<\"3.11\"" ] classifiers = [ From 019adcec9e4c3f4d533ebfff9ff41b86938d1eb2 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 12:08:11 +0800 Subject: [PATCH 19/20] Record the five floors and the closed Dependabot PRs --- docs/updates/2026-09.md | 13 +++++++++++++ docs/updates/README.md | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/updates/2026-09.md b/docs/updates/2026-09.md index 3b399df..64c6826 100644 --- a/docs/updates/2026-09.md +++ b/docs/updates/2026-09.md @@ -108,3 +108,16 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Also**: `mypy automation_file` reported 11 errors in `remote/webdav/client.py`, because `_request(**kwargs: object)` cannot satisfy the requests type stubs. CI does not install those stubs, so it would not have seen the errors. The kwargs are now typed `Any`, and mypy passes with the stubs as well. - **Result**: run for `1874801`: lint and pytest on 3.10–3.14 all pass. - **Open items**: none. + +## U-20260923-09 · 2026-09-23 · Five floors from the Dependabot PRs on main; the PRs closed · #done #deps + +- **What**: five Dependabot PRs still targeted `main` (#91–#95), from before Dependabot was pointed at `dev`. Their floors are applied on `dev`, the same way this morning's four were (U-20260923-01): + - `prometheus_client>=0.26.0` + - `opentelemetry-api>=1.44.0` and `opentelemetry-sdk>=1.44.0` + - `pyarrow>=25.0.1` + - `boxsdk>=10.15.0,<11` + + They are in `dev.toml`, `stable.toml` and `requirements.txt`. `requirements.txt` still had `pyarrow>=15.0.0`, which is now in step with the TOMLs. +- **Checked**: the local venv was upgraded to those versions, `uv pip check` is clean, and the suite gives 759 passed, 8 skipped. +- **PRs**: #91–#95 are closed with a comment pointing at the `dev` commit. +- **Open items**: none. diff --git a/docs/updates/README.md b/docs/updates/README.md index 88187ce..5fd76ff 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260923-09 | 2026-09-23 | Five floors from the Dependabot PRs on main; the PRs closed | #done #deps | [2026-09](2026-09.md) | | U-20260923-08 | 2026-09-23 | dev CI green again: format check and mypy | #done #ci | [2026-09](2026-09.md) | | U-20260923-07 | 2026-09-23 | Stale release/bump-v0.0.32 branch deleted | #done #housekeeping | [2026-09](2026-09.md) | | U-20260923-06 | 2026-09-23 | cryptography floor 50 and msal 1.39 | #done #deps #security | [2026-09](2026-09.md) | @@ -76,4 +77,4 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09.md](2026-09.md) | 2026-09 | 13 | +| [2026-09.md](2026-09.md) | 2026-09 | 14 | From 57f9fb7eafc0d7cbc0b8a2091197982c7c653291 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 23 Sep 2026 12:43:06 +0800 Subject: [PATCH 20/20] Keep docstrings under 100 columns and mark the test subprocess calls for Semgrep --- automation_file/logging_config.py | 7 +++++-- automation_file/remote/box/client.py | 2 +- tests/test_legacy_cli_contract.py | 2 +- tests/test_log_location.py | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/automation_file/logging_config.py b/automation_file/logging_config.py index 8c168e0..577fad5 100644 --- a/automation_file/logging_config.py +++ b/automation_file/logging_config.py @@ -42,7 +42,7 @@ def default_log_file() -> Path: def _rotate_if_large(path: Path, limit: int) -> None: - """Move ``path`` to ``.1`` past ``limit`` bytes; best effort while another process holds it.""" + """Move ``path`` to ``.1`` past ``limit`` bytes; best effort while it is held.""" try: if limit <= 0 or not path.is_file() or path.stat().st_size <= limit: return @@ -52,7 +52,10 @@ def _rotate_if_large(path: Path, limit: int) -> None: class FileAutomationFileHandler(RotatingFileHandler): - """Append-mode UTF-8 file handler; a file that cannot be opened becomes ``os.devnull`` with one warning.""" + """Append-mode UTF-8 file handler. + + A file that cannot be opened becomes ``os.devnull`` with one warning. + """ def __init__(self, filename: str, delay: bool = True) -> None: super().__init__( diff --git a/automation_file/remote/box/client.py b/automation_file/remote/box/client.py index f191a15..86c2f62 100644 --- a/automation_file/remote/box/client.py +++ b/automation_file/remote/box/client.py @@ -60,7 +60,7 @@ def later_init( return self.client def require_client(self) -> Any: - """Return the initialised SDK client, or raise :class:`BoxException` before ``later_init``.""" + """Return the initialised SDK client; raise :class:`BoxException` before ``later_init``.""" if self.client is None: raise BoxException("BoxClient not initialised; call later_init() first") return self.client diff --git a/tests/test_legacy_cli_contract.py b/tests/test_legacy_cli_contract.py index 94c3b1b..c1285f7 100644 --- a/tests/test_legacy_cli_contract.py +++ b/tests/test_legacy_cli_contract.py @@ -29,7 +29,7 @@ def _run_cli(cwd: Path, *args: str) -> subprocess.CompletedProcess: env = dict(os.environ) env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(REPO_ROOT), env.get("PYTHONPATH")])) env["PYTHONIOENCODING"] = "utf-8" - return subprocess.run( # nosec B603 - fixed interpreter, test-controlled arguments + return subprocess.run( # nosemgrep # nosec B603 - fixed interpreter, test arguments [sys.executable, "-m", PACKAGE, *args], cwd=cwd, env=env, diff --git a/tests/test_log_location.py b/tests/test_log_location.py index 6eb299f..f86be9f 100644 --- a/tests/test_log_location.py +++ b/tests/test_log_location.py @@ -1,4 +1,4 @@ -"""Where FileAutomation's log goes, and that importing the package writes nothing (workspace item X-6).""" +"""Where FileAutomation's log goes, and that importing the package writes nothing (X-6).""" import logging import os @@ -49,7 +49,7 @@ def test_importing_writes_no_file(tmp_path): target = tmp_path / "home" / "FileAutomation.log" env = {key: value for key, value in os.environ.items() if key != LOG_FILE_ENV} env[LOG_FILE_ENV] = str(target) - result = subprocess.run( # nosec B603 - fixed interpreter, test-controlled arguments + result = subprocess.run( # nosemgrep # nosec B603 - fixed interpreter, test arguments [sys.executable, "-c", _IMPORT_ONLY, str(MODULE_FILE)], cwd=tmp_path, env=env,