diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index ad00335..98222a3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -20,15 +20,21 @@ concurrency: cancel-in-progress: true jobs: - # The upstream name in a new string literal, typically brought in by an - # upstream merge. Seconds, and needs no toolchain. tools/fork/name-check.py. - name-check: + # What an upstream merge can bring in or leave behind without a conflict: + # the upstream name in a new string literal, and a changed upstream file + # without the AGPL 5(a) notice. Seconds, and needs no toolchain. The notice + # check diffs against the upstream snapshot branch, hence the full fetch. + fork-checks: runs-on: light container: image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim steps: - uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec + with: + fetch-depth: 0 - run: python3 tools/fork/name-check.py + - if: always() + run: python3 tools/fork/notice-check.py build: # Either runner (host1 or host2): the build needs no docker socket. diff --git a/docs/fork/strip-reports/v0.16.23.json b/docs/fork/strip-reports/v0.16.23.json new file mode 100644 index 0000000..d70a579 --- /dev/null +++ b/docs/fork/strip-reports/v0.16.23.json @@ -0,0 +1,500 @@ +{ + "ref": "v0.16.23", + "commit": "9d1c75ab68435e4417337f768291e5f947686203", + "removed_files": [ + "crates/common/src/enterprise/alerts.rs", + "crates/common/src/enterprise/config.rs", + "crates/common/src/enterprise/license.rs", + "crates/common/src/enterprise/llm.rs", + "crates/common/src/enterprise/masked.rs", + "crates/common/src/enterprise/mod.rs", + "crates/common/src/telemetry/metrics/store.rs", + "crates/common/src/telemetry/metrics/test_data.rs", + "crates/common/src/telemetry/tracers/store.rs", + "crates/http/src/api/telemetry.rs", + "crates/jmap/src/registry/mapping/archived_item.rs", + "crates/jmap/src/registry/mapping/masked_email.rs", + "crates/jmap/src/registry/mapping/telemetry.rs", + "crates/scim-proto/src/attributes.rs", + "crates/scim-proto/src/etag.rs", + "crates/scim-proto/src/filter.rs", + "crates/scim-proto/src/json.rs", + "crates/scim-proto/src/lib.rs", + "crates/scim-proto/src/message/bulk.rs", + "crates/scim-proto/src/message/error.rs", + "crates/scim-proto/src/message/list.rs", + "crates/scim-proto/src/message/mod.rs", + "crates/scim-proto/src/message/patch.rs", + "crates/scim-proto/src/message/search.rs", + "crates/scim-proto/src/path.rs", + "crates/scim-proto/src/schema/group.rs", + "crates/scim-proto/src/schema/mod.rs", + "crates/scim-proto/src/schema/spc.rs", + "crates/scim-proto/src/schema/user.rs", + "crates/scim/src/auth.rs", + "crates/scim/src/bulk.rs", + "crates/scim/src/context.rs", + "crates/scim/src/discovery.rs", + "crates/scim/src/error.rs", + "crates/scim/src/groups/get.rs", + "crates/scim/src/groups/mod.rs", + "crates/scim/src/groups/patch.rs", + "crates/scim/src/groups/set.rs", + "crates/scim/src/lib.rs", + "crates/scim/src/query/cursor.rs", + "crates/scim/src/query/mod.rs", + "crates/scim/src/request.rs", + "crates/scim/src/users/get.rs", + "crates/scim/src/users/mod.rs", + "crates/scim/src/users/patch.rs", + "crates/scim/src/users/set.rs", + "crates/spam-filter/src/analysis/llm.rs", + "crates/store/src/backend/composite/mod.rs", + "crates/store/src/backend/composite/read_replica.rs", + "crates/store/src/backend/composite/sharded_blob.rs", + "crates/store/src/backend/composite/sharded_lookup.rs", + "crates/trc/src/serializers/binary.rs", + "tests/src/directory/oidc.rs", + "tests/src/scim/auth.rs", + "tests/src/scim/bulk.rs", + "tests/src/scim/discovery.rs", + "tests/src/scim/groups.rs", + "tests/src/scim/limits.rs", + "tests/src/scim/mod.rs", + "tests/src/scim/query.rs", + "tests/src/scim/users.rs", + "tests/src/system/archiving.rs", + "tests/src/system/tenant.rs" + ], + "removed_snippets": { + "crates/common/src/auth/authentication.rs": 3, + "crates/common/src/auth/mod.rs": 2, + "crates/common/src/auth/permissions.rs": 1, + "crates/common/src/cache/directory.rs": 6, + "crates/common/src/cache/invalidate.rs": 1, + "crates/common/src/cache/principals.rs": 2, + "crates/common/src/cache/reload.rs": 1, + "crates/common/src/config/mod.rs": 2, + "crates/common/src/config/telemetry.rs": 4, + "crates/common/src/lib.rs": 2, + "crates/common/src/manager/boot.rs": 1, + "crates/common/src/network/mta.rs": 1, + "crates/common/src/scripts/plugins/llm_prompt.rs": 1, + "crates/common/src/storage/quota.rs": 2, + "crates/common/src/telemetry/metrics/mod.rs": 1, + "crates/common/src/telemetry/metrics/prometheus.rs": 1, + "crates/common/src/telemetry/mod.rs": 1, + "crates/common/src/telemetry/tracers/mod.rs": 1, + "crates/http/src/api/mod.rs": 4, + "crates/http/src/auth/permissions.rs": 1, + "crates/http/src/request.rs": 4, + "crates/jmap/src/registry/get.rs": 1, + "crates/jmap/src/registry/mapping/mod.rs": 1, + "crates/jmap/src/registry/mapping/principal.rs": 3, + "crates/jmap/src/registry/mapping/queued_message.rs": 1, + "crates/jmap/src/registry/mapping/task.rs": 1, + "crates/jmap/src/registry/mod.rs": 1, + "crates/jmap/src/registry/query.rs": 1, + "crates/jmap/src/registry/set.rs": 2, + "crates/main/src/main.rs": 1, + "crates/services/src/task_manager/alarm.rs": 1, + "crates/services/src/task_manager/imip.rs": 1, + "crates/services/src/task_manager/index.rs": 2, + "crates/services/src/task_manager/maintenance.rs": 3, + "crates/services/src/task_manager/scheduler.rs": 8, + "crates/spam-filter/src/analysis/mod.rs": 1, + "crates/spam-filter/src/analysis/score.rs": 2, + "crates/store/src/backend/mod.rs": 1, + "crates/store/src/backend/mysql/main.rs": 1, + "crates/store/src/backend/postgres/main.rs": 1, + "crates/store/src/build/blob.rs": 2, + "crates/store/src/build/lookup.rs": 1, + "crates/store/src/build/memory.rs": 2, + "crates/store/src/dispatch/blob.rs": 6, + "crates/store/src/dispatch/lookup.rs": 10, + "crates/store/src/dispatch/mod.rs": 1, + "crates/store/src/dispatch/search.rs": 6, + "crates/store/src/dispatch/store.rs": 8, + "crates/store/src/lib.rs": 6, + "crates/trc/src/serializers/mod.rs": 1 + }, + "cargo_edits": [ + { + "file": "crates/main/Cargo.toml", + "line": 50, + "before": "default = [\"rocks\", \"enterprise\"]", + "after": "default = [\"rocks\"]" + }, + { + "file": "tests/Cargo.toml", + "line": 22, + "before": "store = { path = \"../crates/store\", features = [\"test_mode\", \"enterprise\"] }", + "after": "store = { path = \"../crates/store\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 24, + "before": "directory = { path = \"../crates/directory\", features = [\"test_mode\", \"enterprise\"] }", + "after": "directory = { path = \"../crates/directory\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 25, + "before": "coordinator = { path = \"../crates/coordinator\", features = [\"test_mode\", \"enterprise\"] }", + "after": "coordinator = { path = \"../crates/coordinator\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 26, + "before": "jmap = { path = \"../crates/jmap\", features = [\"test_mode\", \"enterprise\"] }", + "after": "jmap = { path = \"../crates/jmap\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 35, + "before": "http = { path = \"../crates/http\", features = [\"test_mode\", \"enterprise\"] }", + "after": "http = { path = \"../crates/http\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 37, + "before": "scim = { path = \"../crates/scim\", features = [\"test_mode\", \"enterprise\"] }", + "after": "scim = { path = \"../crates/scim\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 39, + "before": "services = { path = \"../crates/services\", features = [\"test_mode\", \"enterprise\"] }", + "after": "services = { path = \"../crates/services\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 41, + "before": "smtp = { path = \"../crates/smtp\", features = [\"test_mode\", \"enterprise\"] }", + "after": "smtp = { path = \"../crates/smtp\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 42, + "before": "common = { path = \"../crates/common\", features = [\"test_mode\", \"enterprise\"] }", + "after": "common = { path = \"../crates/common\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 44, + "before": "email = { path = \"../crates/email\", features = [\"test_mode\", \"enterprise\"] }", + "after": "email = { path = \"../crates/email\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 45, + "before": "spam-filter = { path = \"../crates/spam-filter\", features = [\"test_mode\", \"enterprise\"] }", + "after": "spam-filter = { path = \"../crates/spam-filter\", features = [\"test_mode\"] }" + }, + { + "file": "tests/Cargo.toml", + "line": 46, + "before": "trc = { path = \"../crates/trc\", features = [\"enterprise\"] }", + "after": "trc = { path = \"../crates/trc\", features = [] }" + }, + { + "file": "tests/Cargo.toml", + "line": 47, + "before": "managesieve = { path = \"../crates/managesieve\", features = [\"test_mode\", \"enterprise\"] }", + "after": "managesieve = { path = \"../crates/managesieve\", features = [\"test_mode\"] }" + }, + { + "file": ".github/workflows/ci.yml", + "line": 341, + "before": "cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\"", + "after": "cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\"" + }, + { + "file": ".github/workflows/ci.yml", + "line": 379, + "before": "# cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"foundationdb s3 redis nats enterprise\"", + "after": "# cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"foundationdb s3 redis nats\"" + }, + { + "file": ".github/workflows/ci.yml", + "line": 386, + "before": "cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\"", + "after": "cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\"" + }, + { + "file": ".github/workflows/ci.yml", + "line": 442, + "before": "cargo build --release -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\"", + "after": "cargo build --release -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\"" + }, + { + "file": "Dockerfile", + "line": 22, + "before": "RUN RUSTFLAGS=\"$(cat /flags.txt)\" cargo chef cook --target \"$(cat /target.txt)\" --release --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\" --recipe-path /recipe.json", + "after": "RUN RUSTFLAGS=\"$(cat /flags.txt)\" cargo chef cook --target \"$(cat /target.txt)\" --release --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\" --recipe-path /recipe.json" + }, + { + "file": "Dockerfile", + "line": 24, + "before": "RUN RUSTFLAGS=\"$(cat /flags.txt)\" cargo build --target \"$(cat /target.txt)\" --release -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\"", + "after": "RUN RUSTFLAGS=\"$(cat /flags.txt)\" cargo build --target \"$(cat /target.txt)\" --release -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\"" + }, + { + "file": "Dockerfile.build", + "line": 111, + "before": "RUSTFLAGS=\"-L /usr/lib\" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"foundationdb s3 redis nats enterprise\"; \\", + "after": "RUSTFLAGS=\"-L /usr/lib\" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"foundationdb s3 redis nats\"; \\" + }, + { + "file": "Dockerfile.build", + "line": 119, + "before": "cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\"", + "after": "cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\"" + }, + { + "file": "Dockerfile.build", + "line": 132, + "before": "RUSTFLAGS=\"-L /usr/lib\" cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"foundationdb s3 redis nats enterprise\" && \\", + "after": "RUSTFLAGS=\"-L /usr/lib\" cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"foundationdb s3 redis nats\" && \\" + }, + { + "file": "Dockerfile.build", + "line": 142, + "before": "cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats enterprise\" && \\", + "after": "cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features \"sqlite postgres mysql rocks s3 redis azure nats\" && \\" + }, + { + "file": "Dockerfile.fdb", + "line": 56, + "before": "RUN cargo build -p stalwart --no-default-features --features \"foundationdb s3 redis azure nats enterprise\" --release", + "after": "RUN cargo build -p stalwart --no-default-features --features \"foundationdb s3 redis azure nats\" --release" + } + ], + "dangling_mods": [ + { + "file": "crates/common/src/telemetry/metrics/mod.rs", + "line": 12, + "module": "test_data", + "lines": [ + "#[cfg(any(feature = \"dev_mode\", feature = \"test_mode\"))]", + "pub mod test_data;" + ] + }, + { + "file": "tests/src/directory/mod.rs", + "line": 11, + "module": "oidc", + "lines": [ + "pub mod oidc;" + ] + }, + { + "file": "tests/src/lib.rs", + "line": 27, + "module": "scim", + "lines": [ + "#[cfg(test)]", + "pub mod scim;" + ] + }, + { + "file": "tests/src/system/mod.rs", + "line": 8, + "module": "archiving", + "lines": [ + "pub mod archiving;" + ] + }, + { + "file": "tests/src/system/mod.rs", + "line": 19, + "module": "tenant", + "lines": [ + "pub mod tenant;" + ] + } + ], + "problems": [], + "feature_gates": { + "crates/common/src/cache/reload.rs": 1, + "crates/common/src/manager/boot.rs": 1, + "crates/common/src/storage/mod.rs": 1, + "crates/common/src/storage/quota.rs": 1, + "crates/common/src/telemetry/metrics/prometheus.rs": 1, + "crates/http/src/api/mod.rs": 1, + "crates/http/src/auth/permissions.rs": 1, + "crates/jmap/src/registry/get.rs": 1, + "crates/jmap/src/registry/mapping/principal.rs": 2, + "crates/jmap/src/registry/mapping/queued_message.rs": 1, + "crates/jmap/src/registry/mapping/task.rs": 1, + "crates/jmap/src/registry/set.rs": 1, + "crates/services/src/task_manager/alarm.rs": 1, + "crates/services/src/task_manager/imip.rs": 1, + "crates/services/src/task_manager/index.rs": 1, + "crates/services/src/task_manager/maintenance.rs": 1, + "crates/services/src/task_manager/scheduler.rs": 1, + "crates/store/src/lib.rs": 1 + }, + "edition_checks": {}, + "schema": { + "objects": [ + "x:AiModel", + "x:Alert", + "x:ArchivedItem", + "x:MaskedEmail", + "x:MetricsStore", + "x:SpamLlm", + "x:Tenant", + "x:Trace", + "x:TracingStore" + ], + "fields": [ + "x:AcmeProvider.memberTenantId", + "x:ArfExternalReport.memberTenantId", + "x:Authentication.defaultTenantRoleIds", + "x:CalendarAlarm.template", + "x:CalendarScheduling.emailTemplate", + "x:CalendarScheduling.httpRsvpTemplate", + "x:DataRetention.archiveDeletedAccountsFor", + "x:DataRetention.archiveDeletedItemsFor", + "x:DataRetention.holdMetricsFor", + "x:DataRetention.holdTracesFor", + "x:DataRetention.metricsCollectionInterval", + "x:Dkim1Signature.memberTenantId", + "x:Dkim2Signature.memberTenantId", + "x:DmarcExternalReport.memberTenantId", + "x:Domain.allowScimProvisioning", + "x:Domain.directoryId", + "x:Domain.logo", + "x:Domain.memberTenantId", + "x:Email.maxMaskedAddresses", + "x:Enterprise.logoUrl", + "x:GroupAccount.externalId", + "x:LdapDirectory.memberTenantId", + "x:MySqlStore.readReplicas", + "x:OidcDirectory.memberTenantId", + "x:PostgreSqlStore.readReplicas", + "x:Search.indexTelemetry", + "x:Search.indexTracingFields", + "x:SqlDirectory.memberTenantId", + "x:TlsExternalReport.memberTenantId", + "x:UserAccount.externalId" + ] + }, + "third_party": { + "crates/common/src/network/acme/directory.rs": [ + { + "line": 7, + "text": "Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0." + } + ], + "crates/common/src/network/acme/jose.rs": [ + { + "line": 7, + "text": "Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0." + } + ], + "crates/common/src/network/acme/order.rs": [ + { + "line": 7, + "text": "Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0." + } + ], + "crates/common/src/scripts/functions/text.rs": [ + { + "line": 239, + "text": "MIT licensed." + }, + { + "line": 241, + "text": "Copyright (c) 2016 Titus Wormer " + } + ], + "crates/common/src/telemetry/tracers/journald.rs": [ + { + "line": 70, + "text": "SPDX-FileCopyrightText: 2018 Benjamin Saunders " + }, + { + "line": 71, + "text": "SPDX-License-Identifier: MIT" + } + ], + "crates/imap-proto/src/utf7.rs": [ + { + "line": 7, + "text": "Ported from https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs" + } + ], + "crates/jmap/src/registry/mapping/log.rs": [ + { + "line": 341, + "text": "SPDX-FileCopyrightText: 2017 Michael Coyne " + }, + { + "line": 343, + "text": "SPDX-License-Identifier: MIT" + }, + { + "line": 346, + "text": "Adapted from https://github.com/mjc-gh/rev_lines/blob/main/src/lib.rs" + } + ], + "crates/jmap-proto/src/types/date.rs": [ + { + "line": 121, + "text": "Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days" + }, + { + "line": 158, + "text": "Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992" + } + ], + "crates/nlp/src/tokenizers/japanese.rs": [ + { + "line": 73, + "text": "Ported from https://github.com/woxtu/rust-tinysegmenter, MIT license" + } + ], + "crates/nlp/src/tokenizers/types.rs": [ + { + "line": 738, + "text": "Credits: test suite from linkify crate" + } + ], + "crates/registry/src/types/datetime.rs": [ + { + "line": 150, + "text": "Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days" + }, + { + "line": 188, + "text": "Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992" + } + ], + "crates/store/src/backend/postgres/tls.rs": [ + { + "line": 7, + "text": "Credits: https://github.com/jbg/tokio-postgres-rustls" + } + ], + "crates/types/src/id.rs": [ + { + "line": 69, + "text": "From https://github.com/archer884/crockford by J/A " + }, + { + "line": 70, + "text": "License: MIT/Apache 2.0" + } + ], + "crates/utils/src/glob.rs": [ + { + "line": 107, + "text": "Credits: Algorithm ported from https://research.swtch.com/glob" + } + ] + }, + "third_party_unlisted": [], + "ossify_log": "$ ossify.py crates\nProcessing Rust files in: /tmp/claude-1000/-run-media-john-PROJECTS/b60a056e-7e44-433c-bbb6-7e1b1fece570/scratchpad/strip-v0.16.23/tree/crates\n\nFound 957 Rust files\n\n\nSummary:\n- 50 files were completely removed\n- 2 crate roots were emptied\n- 118 proprietary snippets were removed from 50 files\n\n$ ossify.py tests\nProcessing Rust files in: /tmp/claude-1000/-run-media-john-PROJECTS/b60a056e-7e44-433c-bbb6-7e1b1fece570/scratchpad/strip-v0.16.23/tree/tests\n\nFound 211 Rust files\n\n\nSummary:\n- 11 files were completely removed\n- 0 crate roots were emptied\n- 0 proprietary snippets were removed from 0 files\n" +} diff --git a/docs/fork/strip-reports/v0.16.23.md b/docs/fork/strip-reports/v0.16.23.md new file mode 100644 index 0000000..6163abb --- /dev/null +++ b/docs/fork/strip-reports/v0.16.23.md @@ -0,0 +1,211 @@ +# Strip report: upstream v0.16.23 (9d1c75ab6843) + +- Enterprise-only files removed or emptied: **63** +- Enterprise-only snippets removed: **118** in 50 files +- Cargo edits turning `enterprise` off: **25** +- Dangling module declarations removed: **5** +- Verification: **clean** +- Left for the rebuilt features to replace: 19 `enterprise` feature gates in 18 files; 0 `is_enterprise_edition()` checks in 0 files +- Third-party code: 14 files, **0** not in THIRD-PARTY.md +- Upstream schema flags 9 objects and 30 fields as Enterprise + +## Removed files + +- `crates/common/src/enterprise/alerts.rs` +- `crates/common/src/enterprise/config.rs` +- `crates/common/src/enterprise/license.rs` +- `crates/common/src/enterprise/llm.rs` +- `crates/common/src/enterprise/masked.rs` +- `crates/common/src/enterprise/mod.rs` +- `crates/common/src/telemetry/metrics/store.rs` +- `crates/common/src/telemetry/metrics/test_data.rs` +- `crates/common/src/telemetry/tracers/store.rs` +- `crates/http/src/api/telemetry.rs` +- `crates/jmap/src/registry/mapping/archived_item.rs` +- `crates/jmap/src/registry/mapping/masked_email.rs` +- `crates/jmap/src/registry/mapping/telemetry.rs` +- `crates/scim-proto/src/attributes.rs` +- `crates/scim-proto/src/etag.rs` +- `crates/scim-proto/src/filter.rs` +- `crates/scim-proto/src/json.rs` +- `crates/scim-proto/src/lib.rs` +- `crates/scim-proto/src/message/bulk.rs` +- `crates/scim-proto/src/message/error.rs` +- `crates/scim-proto/src/message/list.rs` +- `crates/scim-proto/src/message/mod.rs` +- `crates/scim-proto/src/message/patch.rs` +- `crates/scim-proto/src/message/search.rs` +- `crates/scim-proto/src/path.rs` +- `crates/scim-proto/src/schema/group.rs` +- `crates/scim-proto/src/schema/mod.rs` +- `crates/scim-proto/src/schema/spc.rs` +- `crates/scim-proto/src/schema/user.rs` +- `crates/scim/src/auth.rs` +- `crates/scim/src/bulk.rs` +- `crates/scim/src/context.rs` +- `crates/scim/src/discovery.rs` +- `crates/scim/src/error.rs` +- `crates/scim/src/groups/get.rs` +- `crates/scim/src/groups/mod.rs` +- `crates/scim/src/groups/patch.rs` +- `crates/scim/src/groups/set.rs` +- `crates/scim/src/lib.rs` +- `crates/scim/src/query/cursor.rs` +- `crates/scim/src/query/mod.rs` +- `crates/scim/src/request.rs` +- `crates/scim/src/users/get.rs` +- `crates/scim/src/users/mod.rs` +- `crates/scim/src/users/patch.rs` +- `crates/scim/src/users/set.rs` +- `crates/spam-filter/src/analysis/llm.rs` +- `crates/store/src/backend/composite/mod.rs` +- `crates/store/src/backend/composite/read_replica.rs` +- `crates/store/src/backend/composite/sharded_blob.rs` +- `crates/store/src/backend/composite/sharded_lookup.rs` +- `crates/trc/src/serializers/binary.rs` +- `tests/src/directory/oidc.rs` +- `tests/src/scim/auth.rs` +- `tests/src/scim/bulk.rs` +- `tests/src/scim/discovery.rs` +- `tests/src/scim/groups.rs` +- `tests/src/scim/limits.rs` +- `tests/src/scim/mod.rs` +- `tests/src/scim/query.rs` +- `tests/src/scim/users.rs` +- `tests/src/system/archiving.rs` +- `tests/src/system/tenant.rs` + +## Removed snippets + +- `crates/common/src/auth/authentication.rs`: 3 +- `crates/common/src/auth/mod.rs`: 2 +- `crates/common/src/auth/permissions.rs`: 1 +- `crates/common/src/cache/directory.rs`: 6 +- `crates/common/src/cache/invalidate.rs`: 1 +- `crates/common/src/cache/principals.rs`: 2 +- `crates/common/src/cache/reload.rs`: 1 +- `crates/common/src/config/mod.rs`: 2 +- `crates/common/src/config/telemetry.rs`: 4 +- `crates/common/src/lib.rs`: 2 +- `crates/common/src/manager/boot.rs`: 1 +- `crates/common/src/network/mta.rs`: 1 +- `crates/common/src/scripts/plugins/llm_prompt.rs`: 1 +- `crates/common/src/storage/quota.rs`: 2 +- `crates/common/src/telemetry/metrics/mod.rs`: 1 +- `crates/common/src/telemetry/metrics/prometheus.rs`: 1 +- `crates/common/src/telemetry/mod.rs`: 1 +- `crates/common/src/telemetry/tracers/mod.rs`: 1 +- `crates/http/src/api/mod.rs`: 4 +- `crates/http/src/auth/permissions.rs`: 1 +- `crates/http/src/request.rs`: 4 +- `crates/jmap/src/registry/get.rs`: 1 +- `crates/jmap/src/registry/mapping/mod.rs`: 1 +- `crates/jmap/src/registry/mapping/principal.rs`: 3 +- `crates/jmap/src/registry/mapping/queued_message.rs`: 1 +- `crates/jmap/src/registry/mapping/task.rs`: 1 +- `crates/jmap/src/registry/mod.rs`: 1 +- `crates/jmap/src/registry/query.rs`: 1 +- `crates/jmap/src/registry/set.rs`: 2 +- `crates/main/src/main.rs`: 1 +- `crates/services/src/task_manager/alarm.rs`: 1 +- `crates/services/src/task_manager/imip.rs`: 1 +- `crates/services/src/task_manager/index.rs`: 2 +- `crates/services/src/task_manager/maintenance.rs`: 3 +- `crates/services/src/task_manager/scheduler.rs`: 8 +- `crates/spam-filter/src/analysis/mod.rs`: 1 +- `crates/spam-filter/src/analysis/score.rs`: 2 +- `crates/store/src/backend/mod.rs`: 1 +- `crates/store/src/backend/mysql/main.rs`: 1 +- `crates/store/src/backend/postgres/main.rs`: 1 +- `crates/store/src/build/blob.rs`: 2 +- `crates/store/src/build/lookup.rs`: 1 +- `crates/store/src/build/memory.rs`: 2 +- `crates/store/src/dispatch/blob.rs`: 6 +- `crates/store/src/dispatch/lookup.rs`: 10 +- `crates/store/src/dispatch/mod.rs`: 1 +- `crates/store/src/dispatch/search.rs`: 6 +- `crates/store/src/dispatch/store.rs`: 8 +- `crates/store/src/lib.rs`: 6 +- `crates/trc/src/serializers/mod.rs`: 1 + +## Dangling module declarations removed + +- `crates/common/src/telemetry/metrics/mod.rs:12`: `mod test_data` (#[cfg(any(feature = "dev_mode", feature = "test_mode"))] / pub mod test_data;) +- `tests/src/directory/mod.rs:11`: `mod oidc` (pub mod oidc;) +- `tests/src/lib.rs:27`: `mod scim` (#[cfg(test)] / pub mod scim;) +- `tests/src/system/mod.rs:8`: `mod archiving` (pub mod archiving;) +- `tests/src/system/mod.rs:19`: `mod tenant` (pub mod tenant;) + +## Cargo edits + +- `crates/main/Cargo.toml:50`: `default = ["rocks", "enterprise"]` → `default = ["rocks"]` +- `tests/Cargo.toml:22`: `store = { path = "../crates/store", features = ["test_mode", "enterprise"] }` → `store = { path = "../crates/store", features = ["test_mode"] }` +- `tests/Cargo.toml:24`: `directory = { path = "../crates/directory", features = ["test_mode", "enterprise"] }` → `directory = { path = "../crates/directory", features = ["test_mode"] }` +- `tests/Cargo.toml:25`: `coordinator = { path = "../crates/coordinator", features = ["test_mode", "enterprise"] }` → `coordinator = { path = "../crates/coordinator", features = ["test_mode"] }` +- `tests/Cargo.toml:26`: `jmap = { path = "../crates/jmap", features = ["test_mode", "enterprise"] }` → `jmap = { path = "../crates/jmap", features = ["test_mode"] }` +- `tests/Cargo.toml:35`: `http = { path = "../crates/http", features = ["test_mode", "enterprise"] }` → `http = { path = "../crates/http", features = ["test_mode"] }` +- `tests/Cargo.toml:37`: `scim = { path = "../crates/scim", features = ["test_mode", "enterprise"] }` → `scim = { path = "../crates/scim", features = ["test_mode"] }` +- `tests/Cargo.toml:39`: `services = { path = "../crates/services", features = ["test_mode", "enterprise"] }` → `services = { path = "../crates/services", features = ["test_mode"] }` +- `tests/Cargo.toml:41`: `smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] }` → `smtp = { path = "../crates/smtp", features = ["test_mode"] }` +- `tests/Cargo.toml:42`: `common = { path = "../crates/common", features = ["test_mode", "enterprise"] }` → `common = { path = "../crates/common", features = ["test_mode"] }` +- `tests/Cargo.toml:44`: `email = { path = "../crates/email", features = ["test_mode", "enterprise"] }` → `email = { path = "../crates/email", features = ["test_mode"] }` +- `tests/Cargo.toml:45`: `spam-filter = { path = "../crates/spam-filter", features = ["test_mode", "enterprise"] }` → `spam-filter = { path = "../crates/spam-filter", features = ["test_mode"] }` +- `tests/Cargo.toml:46`: `trc = { path = "../crates/trc", features = ["enterprise"] }` → `trc = { path = "../crates/trc", features = [] }` +- `tests/Cargo.toml:47`: `managesieve = { path = "../crates/managesieve", features = ["test_mode", "enterprise"] }` → `managesieve = { path = "../crates/managesieve", features = ["test_mode"] }` +- `.github/workflows/ci.yml:341`: `cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise"` → `cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"` +- `.github/workflows/ci.yml:379`: `# cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise"` → `# cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "foundationdb s3 redis nats"` +- `.github/workflows/ci.yml:386`: `cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise"` → `cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"` +- `.github/workflows/ci.yml:442`: `cargo build --release -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise"` → `cargo build --release -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"` +- `Dockerfile:22`: `RUN RUSTFLAGS="$(cat /flags.txt)" cargo chef cook --target "$(cat /target.txt)" --release --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" --recipe-path /recipe.json` → `RUN RUSTFLAGS="$(cat /flags.txt)" cargo chef cook --target "$(cat /target.txt)" --release --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats" --recipe-path /recipe.json` +- `Dockerfile:24`: `RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise"` → `RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"` +- `Dockerfile.build:111`: `RUSTFLAGS="-L /usr/lib" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise"; \` → `RUSTFLAGS="-L /usr/lib" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats"; \` +- `Dockerfile.build:119`: `cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise"` → `cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"` +- `Dockerfile.build:132`: `RUSTFLAGS="-L /usr/lib" cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise" && \` → `RUSTFLAGS="-L /usr/lib" cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats" && \` +- `Dockerfile.build:142`: `cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" && \` → `cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats" && \` +- `Dockerfile.fdb:56`: `RUN cargo build -p stalwart --no-default-features --features "foundationdb s3 redis azure nats enterprise" --release` → `RUN cargo build -p stalwart --no-default-features --features "foundationdb s3 redis azure nats" --release` + +## Flagged Enterprise in upstream's schema + +**Objects:** `x:AiModel`, `x:Alert`, `x:ArchivedItem`, `x:MaskedEmail`, `x:MetricsStore`, `x:SpamLlm`, `x:Tenant`, `x:Trace`, `x:TracingStore` + +**Fields:** `x:AcmeProvider.memberTenantId`, `x:ArfExternalReport.memberTenantId`, `x:Authentication.defaultTenantRoleIds`, `x:CalendarAlarm.template`, `x:CalendarScheduling.emailTemplate`, `x:CalendarScheduling.httpRsvpTemplate`, `x:DataRetention.archiveDeletedAccountsFor`, `x:DataRetention.archiveDeletedItemsFor`, `x:DataRetention.holdMetricsFor`, `x:DataRetention.holdTracesFor`, `x:DataRetention.metricsCollectionInterval`, `x:Dkim1Signature.memberTenantId`, `x:Dkim2Signature.memberTenantId`, `x:DmarcExternalReport.memberTenantId`, `x:Domain.allowScimProvisioning`, `x:Domain.directoryId`, `x:Domain.logo`, `x:Domain.memberTenantId`, `x:Email.maxMaskedAddresses`, `x:Enterprise.logoUrl`, `x:GroupAccount.externalId`, `x:LdapDirectory.memberTenantId`, `x:MySqlStore.readReplicas`, `x:OidcDirectory.memberTenantId`, `x:PostgreSqlStore.readReplicas`, `x:Search.indexTelemetry`, `x:Search.indexTracingFields`, `x:SqlDirectory.memberTenantId`, `x:TlsExternalReport.memberTenantId`, `x:UserAccount.externalId` + +## Third-party code + +Comments in the stripped tree that name another copyright holder, another license, or a source the code came from. Files marked **new** aren't in THIRD-PARTY.md yet. + +- `crates/common/src/network/acme/directory.rs` + - 7: Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. +- `crates/common/src/network/acme/jose.rs` + - 7: Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. +- `crates/common/src/network/acme/order.rs` + - 7: Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. +- `crates/common/src/scripts/functions/text.rs` + - 239: MIT licensed. + - 241: Copyright (c) 2016 Titus Wormer +- `crates/common/src/telemetry/tracers/journald.rs` + - 70: SPDX-FileCopyrightText: 2018 Benjamin Saunders + - 71: SPDX-License-Identifier: MIT +- `crates/imap-proto/src/utf7.rs` + - 7: Ported from https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs +- `crates/jmap/src/registry/mapping/log.rs` + - 341: SPDX-FileCopyrightText: 2017 Michael Coyne + - 343: SPDX-License-Identifier: MIT + - 346: Adapted from https://github.com/mjc-gh/rev_lines/blob/main/src/lib.rs +- `crates/jmap-proto/src/types/date.rs` + - 121: Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days + - 158: Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992 +- `crates/nlp/src/tokenizers/japanese.rs` + - 73: Ported from https://github.com/woxtu/rust-tinysegmenter, MIT license +- `crates/nlp/src/tokenizers/types.rs` + - 738: Credits: test suite from linkify crate +- `crates/registry/src/types/datetime.rs` + - 150: Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days + - 188: Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992 +- `crates/store/src/backend/postgres/tls.rs` + - 7: Credits: https://github.com/jbg/tokio-postgres-rustls +- `crates/types/src/id.rs` + - 69: From https://github.com/archer884/crockford by J/A + - 70: License: MIT/Apache 2.0 +- `crates/utils/src/glob.rs` + - 107: Credits: Algorithm ported from https://research.swtch.com/glob diff --git a/docs/spec/SPEC.md b/docs/spec/SPEC.md index ac7b1c0..cddc043 100644 --- a/docs/spec/SPEC.md +++ b/docs/spec/SPEC.md @@ -91,7 +91,22 @@ The wrapper is `tools/fork/strip.py`. Beyond `ossify.py` it: was ported or adapted from elsewhere. Any file `THIRD-PARTY.md` doesn't cover yet is flagged as new. It's reported, not a failure: the notice goes into `THIRD-PARTY.md` in the merge that brings the release in, since the - fork redistributes that code and its license requires the notice. + fork redistributes that code and its license requires the notice; +- renames the upstream name where it's an identifier clients, users or + operators meet (wire-protocol names, the web interface's client id, store + keys; §2.4), with the same substitutions `main` + carries, so those lines arrive purged and never conflict (added + 2026-09-22); +- compiles the stripped tree. A dual-licensed file that only serves an + Enterprise feature survives the strip but can't build without it: + v0.16.23's `tests/src/directory/issuer.rs` was the first. The build check + fails the run on it, and the merge into `main` drops or reworks it (added + 2026-09-22). + +Two checks in CI cover what a merge can bring in without a conflict: +`tools/fork/name-check.py` (the upstream name in a new string literal) and +`tools/fork/notice-check.py` (a changed upstream file without its AGPL 5(a) +notice). ### 2.2a Snapshots, not a git fork diff --git a/tools/fork/README.md b/tools/fork/README.md index 0821ac0..e20d37d 100644 --- a/tools/fork/README.md +++ b/tools/fork/README.md @@ -19,7 +19,23 @@ The report's Third-party code section lists upstream code under other licenses. Files marked **new** need their notice added to `THIRD-PARTY.md` at the repository root before the import is merged. -It needs Python 3.12+ (for `tarfile`'s `data` filter) and git. +It needs Python 3.12+ (for `tarfile`'s `data` filter), git and cargo. + +Two passes run after the strip: + +- **Renames.** The upstream name is replaced where it's an identifier + clients, users or operators meet: wire-protocol names, the web interface's + client id, store keys, configuration defaults and the served schema, as + `renames.py` lists them. `main` was renamed with the same module. A + re-import arrives purged, so those lines never conflict. Copyright notices + and prose are left alone. The report lists every substitution by file. +- **Build check.** The stripped tree is compiled (`cargo check --workspace + --all-targets`) into `target/strip-check`, which stays warm between + imports. A file that survived the strip but calls code that didn't fails + the run; the report names it. Handle it in the merge into `main`, never on + `upstream`: `upstream` holds the strip's output and nothing else. Imports + the strip left unused are listed without failing. `--no-build-check` + skips the pass. ## name-check.py @@ -37,6 +53,21 @@ Rename what it reports. If a string has to stay, such as a key-derivation context or a wire-protocol identifier, add its `--list` line to the allowlist under the reason it stays. +## notice-check.py + +Fails when an upstream file the fork changed doesn't carry the AGPL 5(a) +notice, `Modified by Coffey Labs in for INBUXA.`, under upstream's +license line. "Changed" means it differs from the `upstream` branch, so the +list comes from the diff, not from memory. CI runs it beside the name check. + +```bash +tools/fork/notice-check.py # exit 1 on a missing notice +tools/fork/notice-check.py --fix # add it where it's missing +``` + +Run `--fix` after resolving an upstream merge: a conflict resolved by taking +upstream's side can drop a notice the file had. + ## record-compat.py Records what the `*_compat` tests compare against, from the Enterprise diff --git a/tools/fork/build-check-known.txt b/tools/fork/build-check-known.txt new file mode 100644 index 0000000..ada8133 --- /dev/null +++ b/tools/fork/build-check-known.txt @@ -0,0 +1,18 @@ +# Files in a stripped upstream tree that are expected not to compile, read by +# tools/fork/strip.py's build check. Each is upstream's shared test of a +# feature upstream builds only in Enterprise and the fork rebuilt clean-room +# on `main` (docs/spec/SPEC.md §2.2b, §4), so the stripped tree alone lacks +# what they call. Errors here are reported as expected; an error in any other +# file fails the strip. One path per line. + +# OIDC directories: `main` has its own tests/src/directory/oidc.rs. +tests/src/directory/mod.rs +# Tenants and archiving. +tests/src/system/mod.rs +# The LLM spam-filter classifier. +tests/src/smtp/inbound/antispam.rs +# Telemetry: alerts, stored metrics and traces, webhooks. +tests/src/telemetry/alerts.rs +tests/src/telemetry/metrics.rs +tests/src/telemetry/tracing.rs +tests/src/telemetry/webhooks.rs diff --git a/tools/fork/notice-check.py b/tools/fork/notice-check.py new file mode 100755 index 0000000..2a3b517 --- /dev/null +++ b/tools/fork/notice-check.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +""" +Fail when an upstream file the fork changed doesn't say so (AGPL section 5(a)). + + tools/fork/notice-check.py # check; exit 1 on a missing notice + tools/fork/notice-check.py --fix # add the notice where it's missing + +The AGPL asks a modified work to carry a prominent notice that it was +modified, with a date. Every upstream file this fork changes carries one +beneath upstream's own notice: + + * Modified by Coffey Labs in 2026 for INBUXA. + +"Changed" is measured against the `upstream` branch, which holds the stripped +upstream release `main` was last merged with (docs/spec/SPEC.md §2.2a), so +the list is what actually differs rather than a guess. A file counts as +upstream's when its header names Stalwart Labs as a copyright holder; files +the fork wrote carry their own copyright and need nothing. Files with no +comment header at all (README, manifests) are covered by the README's prose. +""" +import argparse +import datetime +import re +import subprocess +import sys + +HEADER_LINES = 15 +UPSTREAM_HOLDER = re.compile(r'SPDX-FileCopyrightText:.*Stalwart Labs') +NOTICE = re.compile(r'Modified by Coffey Labs in \d{4}') +LICENSE_LINE = re.compile(r'^(\s*(?:\*|//|#)\s*)SPDX-License-Identifier:.*$') + + +def git(*args): + return subprocess.run(['git', *args], check=True, capture_output=True, text=True).stdout + + +def snapshot_ref(): + for ref in ('origin/upstream', 'upstream'): + if subprocess.run(['git', 'rev-parse', '--verify', '--quiet', f'{ref}^{{commit}}'], + capture_output=True).returncode == 0: + return ref + sys.exit('notice-check: no upstream snapshot branch (origin/upstream or upstream); fetch it first') + + +def changed_upstream_files(ref): + # Against the working tree, not HEAD, so it also checks work not yet + # committed; in CI the two are the same. + for path in git('diff', '--name-only', '--diff-filter=M', ref).splitlines(): + try: + head = open(path, encoding='utf-8').read().split('\n')[:HEADER_LINES] + except (OSError, UnicodeDecodeError): + continue + if any(UPSTREAM_HOLDER.search(line) for line in head): + yield path, head + + +def add_notice(path): + """Put the notice under the license line, in that comment's own style.""" + lines = open(path, encoding='utf-8').read().split('\n') + for n, line in enumerate(lines[:HEADER_LINES]): + m = LICENSE_LINE.match(line) + if m: + prefix = m.group(1) + blank = prefix.rstrip() + year = datetime.date.today().year + lines[n + 1:n + 1] = [blank, f'{prefix}Modified by Coffey Labs in {year} for INBUXA.'] + open(path, 'w', encoding='utf-8').write('\n'.join(lines)) + return True + return False + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0].strip()) + ap.add_argument('--fix', action='store_true', help='add the notice to every file missing it') + args = ap.parse_args() + + ref = snapshot_ref() + checked, missing = 0, [] + for path, head in changed_upstream_files(ref): + checked += 1 + if not any(NOTICE.search(line) for line in head): + missing.append(path) + + if args.fix: + unfixable = [p for p in missing if not add_notice(p)] + for p in sorted(set(missing) - set(unfixable)): + print(f'added: {p}') + missing = unfixable + + if missing: + print(f'{len(missing)} upstream file(s) changed against {ref} without the modification notice:\n') + for p in missing: + print(f' {p}') + print('\nAdd "Modified by Coffey Labs in for INBUXA." under the license line, ' + 'or run tools/fork/notice-check.py --fix.') + return 1 + print(f'notice check: clean ({checked} changed upstream file(s), all marked; against {ref}).') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/fork/renames.py b/tools/fork/renames.py new file mode 100644 index 0000000..db38d9d --- /dev/null +++ b/tools/fork/renames.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs +# SPDX-License-Identifier: AGPL-3.0-only +""" +The upstream name where it's an identifier, and its replacement. + + tools/fork/renames.py DIR # apply to a tree; prints what changed + +Clients, users and operators meet the upstream name in a few places that +aren't notices: wire-protocol names (the JMAP registry capability, WebDAV +state tokens, Sieve extensions), the web interface's OAuth client id, store keys an +operator sees as blob names, and configuration defaults (SQL database and +user, the log file prefix). docs/spec/SPEC.md §2.4 renames them all. + +strip.py applies this to every upstream import, so each release arrives +already renamed and those lines never conflict in the merge. `main` was +renamed with it once, on 2026-09-22. Don't run it on `main` again: code +written since that names the old spelling on purpose (the migrations that +retire it) would be renamed too. Those strings are listed in +name-allowlist.txt instead. + +Names nobody sees keep upstream's spelling: the OAuth key-derivation contexts +and the hashed application prefix. Renaming them would only destroy state. +Copyright notices and prose (`.md`, `.txt`) are never touched. +""" +import gzip +import hashlib +import base64 +import re +import sys +from pathlib import Path + +# Plain substrings, in code and data: (old, new, the roots it applies under). +TEXT_RENAMES = [ + # Upstream's JMAP capability for its registry (`x:`) objects. Not plain + # `urn:inbuxa:jmap`, which is the fork's own capability (contract C-1); + # listed before the general prefix below so it wins. + ('urn:stalwart:jmap', 'urn:inbuxa:jmap:registry', None), + ('urn:stalwart:', 'urn:inbuxa:', None), + ('vnd.stalwart.', 'vnd.inbuxa.', None), + ('(vnd.stalwart)', '(vnd.inbuxa)', None), + ('stalwart-webui', 'inbuxa-webui', None), + ('STALWART_SPAM_', 'INBUXA_SPAM_', None), + # Configuration defaults, as upstream's generated registry code spells + # them. Server code only: the tests use the same spelling for fixtures + # that must match their containers and identity provider (database users, + # passwords, an OIDC audience), and name their databases explicitly. + ('"stalwart".to_string()', '"inbuxa".to_string()', ('crates',)), +] +ROOTS = ('crates', 'tests', 'resources') +SKIP_SUFFIXES = {'.md', '.txt'} +TEXT_SUFFIXES = {'.rs', '.toml', '.py', '.sh', '.json', '.yml', '.yaml', '.js', '.ts', '.html', '.sieve', '.sql'} +COPYRIGHT = re.compile(r'(?i)(?:SPDX-FileCopyrightText:|\bcopyright\b|©)') + +# The JSON Schema the server serves to INBUXA Admin, and its checksum. +SCHEMA = Path('resources/schema/schema.json.gz') +SCHEMA_HASH = Path('resources/schema/schema.json.sha256') +SCHEMA_RENAMES = [ + ('"stalwart"', '"inbuxa"'), + ('vnd.stalwart', 'vnd.inbuxa'), +] + + +def rename_line(line, root): + """Returns (new line, [(old, new, count)]) for a line of a file under `root`.""" + if COPYRIGHT.search(line): + return line, [] + done = [] + for old, new, roots in TEXT_RENAMES: + if roots is not None and root not in roots: + continue + if old in line: + done.append((old, new, line.count(old))) + line = line.replace(old, new) + return line, done + + +def schema_bytes(gz): + """The renamed schema, gzipped deterministically, and its checksum.""" + text = gzip.decompress(gz).decode('utf-8') + for old, new in SCHEMA_RENAMES: + text = text.replace(old, new) + out = gzip.compress(text.encode('utf-8'), compresslevel=9, mtime=0) + digest = base64.urlsafe_b64encode(hashlib.sha256(out).digest()).decode().rstrip('=') + return out, digest + + +def apply(tree): + """Apply every rename under `tree`; returns {"old → new": {file: count}}.""" + tree = Path(tree) + done = {} + + def note(old, new, rel, count): + done.setdefault(f'{old} → {new}', {}) + done[f'{old} → {new}'][rel] = done[f'{old} → {new}'].get(rel, 0) + count + + for root in ROOTS: + base = tree / root + if not base.is_dir(): + continue + for path in sorted(base.rglob('*')): + if not path.is_file() or path.suffix in SKIP_SUFFIXES: + continue + if path.suffix not in TEXT_SUFFIXES and not path.name.startswith('Dockerfile'): + continue + try: + lines = path.read_text(encoding='utf-8').split('\n') + except UnicodeDecodeError: + continue + changed = False + rel = str(path.relative_to(tree)) + for n, line in enumerate(lines): + new_line, subs = rename_line(line, root) + for old, new, count in subs: + note(old, new, rel, count) + if subs: + lines[n] = new_line + changed = True + if changed: + path.write_text('\n'.join(lines), encoding='utf-8') + + schema = tree / SCHEMA + if schema.is_file(): + before = schema.read_bytes() + text = gzip.decompress(before).decode('utf-8') + counts = {old: text.count(old) for old, _ in SCHEMA_RENAMES} + out, digest = schema_bytes(before) + if any(counts.values()): + schema.write_bytes(out) + (tree / SCHEMA_HASH).write_text(digest, encoding='utf-8') + for old, new in SCHEMA_RENAMES: + if counts[old]: + note(old, new, str(SCHEMA), counts[old]) + return {k: dict(sorted(v.items())) for k, v in sorted(done.items())} + + +if __name__ == '__main__': + if len(sys.argv) != 2: + sys.exit(__doc__.split('\n\n')[1]) + for sub, files in apply(sys.argv[1]).items(): + print(f'{sub}: ' + ', '.join(f'{f} ({n})' for f, n in files.items())) diff --git a/tools/fork/strip.py b/tools/fork/strip.py index 2fd5232..575f9cc 100755 --- a/tools/fork/strip.py +++ b/tools/fork/strip.py @@ -33,6 +33,15 @@ What it does, in order (docs/spec/SPEC.md §2.2): comments mark it (another copyright holder or license, or "ported from" and the like), and names any file THIRD-PARTY.md doesn't cover yet. That's a report, not a failure: the notice goes in THIRD-PARTY.md with the merge. +8. Renames the upstream name where it's an identifier clients, users or + operators meet (renames.py beside this script), so a re-import arrives + purged and merges without conflicts on those lines. Copyright notices + and prose are never touched. +9. Compiles the result (`cargo check --workspace --all-targets`). A file + that survived the strip but calls code that didn't -- a dual-licensed test + of an Enterprise feature, say -- fails here, on the `upstream` branch, + instead of in the merge. Imports the strip left unused are reported, not + failed. `--no-build-check` skips it. Only license markers and Cargo manifests are read for meaning. The code inside an Enterprise file or snippet is never printed, reported or kept, which is what @@ -52,6 +61,9 @@ import tarfile import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from renames import apply as rename_upstream_names # noqa: E402 + SEL = 'LicenseRef-SEL' AGPL = 'AGPL-3.0-only' # A license identifier as it appears in a comment header, in any comment style. @@ -375,6 +387,50 @@ def remaining_hooks(tree): return dict(sorted(gates.items())), dict(sorted(checks.items())) +BUILD_KNOWN = Path(__file__).resolve().parent / 'build-check-known.txt' + + +def known_build_failures(): + """Files expected not to compile in a stripped tree (build-check-known.txt).""" + if not BUILD_KNOWN.is_file(): + return set() + return {l.strip() for l in BUILD_KNOWN.read_text(encoding='utf-8').split('\n') + if l.strip() and not l.lstrip().startswith('#')} + + +def build_check(tree, target_dir): + """ + `cargo check` the stripped tree. Returns (errors, unused): each error is + {file, line, message} from a compiler diagnostic, grouped by the file it + points at; unused lists the imports reported unused. + + Only compiler diagnostics are read, and they point at the shared code + that failed, never at the removed code. + """ + cmd = ['cargo', 'check', '--workspace', '--all-targets', '--locked', '--message-format=json', + '--target-dir', str(target_dir)] + r = subprocess.run(cmd, cwd=tree, capture_output=True, text=True) + errors, unused = [], [] + for line in r.stdout.splitlines(): + try: + msg = json.loads(line) + except ValueError: + continue + if msg.get('reason') != 'compiler-message': + continue + d = msg['message'] + span = next((s for s in d.get('spans', []) if s.get('is_primary')), None) + where = {'file': span['file_name'], 'line': span['line_start']} if span else {'file': '?', 'line': 0} + if d.get('level') == 'error': + errors.append({**where, 'message': d.get('message', '')}) + elif (d.get('code') or {}).get('code') == 'unused_imports': + unused.append({**where, 'message': d.get('message', '')}) + if r.returncode != 0 and not errors: + errors.append({'file': '?', 'line': 0, 'message': (r.stderr.strip().splitlines() or ['cargo check failed'])[-1]}) + dedup = lambda items: [dict(t) for t in sorted({tuple(sorted(i.items())) for i in items}, key=lambda t: (dict(t)['file'], dict(t)['line']))] + return dedup(errors), dedup(unused) + + def write_report(out_dir, report): (out_dir / 'STRIP-REPORT.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') r = report @@ -390,6 +446,13 @@ def write_report(out_dir, report): f'in {len(r["feature_gates"])} files; {sum(r["edition_checks"].values())} `is_enterprise_edition()` checks ' f'in {len(r["edition_checks"])} files', f'- Third-party code: {len(r["third_party"])} files, **{len(r["third_party_unlisted"])}** not in THIRD-PARTY.md', + f'- Renamed identifiers: {sum(sum(f.values()) for f in r["renames"].values())} in ' + f'{len({p for f in r["renames"].values() for p in f})} files', + '- Build check: ' + ('skipped' if r['build'] is None else + f'**{"clean" if not r["build"]["errors"] else f"{len(r["build"]["errors"])} errors"}**, ' + f'{len(r["build"]["expected"])} expected errors in ' + f'{len({e["file"] for e in r["build"]["expected"]})} rebuilt-feature tests, ' + f'{len(r["build"]["unused"])} imports left unused'), ] if r['schema']: md.append(f'- Upstream schema flags {len(r["schema"]["objects"])} objects and {len(r["schema"]["fields"])} fields as Enterprise') @@ -406,6 +469,19 @@ def write_report(out_dir, report): for f, found in r['third_party'].items(): md.append(f'- `{f}`{" **new**" if f in r["third_party_unlisted"] else ""}') md += [f' - {h["line"]}: {h["text"]}' for h in found] + md += ['', '## Renamed identifiers', ''] + for sub, files in r['renames'].items(): + md.append(f'- `{sub}`: ' + ', '.join(f'`{f}` ({n})' for f, n in files.items())) + if r['build'] is not None: + md += ['', '## Build check', '', + 'Errors mean shared code calls something the strip removed: usually a dual-licensed file that only ' + 'serves an Enterprise feature. Drop or rework it in the merge into `main`, never on `upstream`.', ''] + md += [f'- error `{e["file"]}:{e["line"]}`: {e["message"]}' for e in r['build']['errors']] + md += [f'- unused `{u["file"]}:{u["line"]}`: {u["message"]}' for u in r['build']['unused']] + md += ['', 'Expected: upstream\'s tests of features the fork rebuilt on `main` ' + '(tools/fork/build-check-known.txt).', ''] + md += [f'- `{e["file"]}:{e["line"]}`: {e["message"]}' for e in r['build']['expected']] + md += [f'- `{f}` now compiles: take it off the known list' for f in r['build']['known_clean']] if r['problems']: md += ['', '## Problems', ''] + [f'- {p}' for p in r['problems']] (out_dir / 'STRIP-REPORT.md').write_text('\n'.join(md) + '\n', encoding='utf-8') @@ -416,6 +492,10 @@ def main(): ap.add_argument('--upstream', required=True, type=Path, help='a git clone of upstream Stalwart') ap.add_argument('--ref', required=True, help='tag, branch or commit to snapshot, e.g. v0.16.22') ap.add_argument('--out', required=True, type=Path, help='new directory; the tree goes in OUT/tree') + ap.add_argument('--target-dir', type=Path, default=Path(__file__).resolve().parents[2] / 'target' / 'strip-check', + help="cargo's target dir for the build check (default: this repo's target/strip-check, " + 'which keeps the dependency build warm between imports)') + ap.add_argument('--no-build-check', action='store_true', help='skip compiling the stripped tree') args = ap.parse_args() if args.out.exists(): @@ -428,7 +508,7 @@ def main(): write_report(args.out, {'ref': args.ref, 'commit': commit, 'removed_files': [], 'removed_snippets': {}, 'cargo_edits': [], 'dangling_mods': [], 'problems': malformed, 'feature_gates': {}, 'edition_checks': {}, 'schema': None, 'third_party': {}, 'third_party_unlisted': [], - 'ossify_log': ''}) + 'renames': {}, 'build': None, 'ossify_log': ''}) print('\n'.join(malformed), file=sys.stderr) fail('malformed snippet markers; nothing stripped', code=1) @@ -436,10 +516,21 @@ def main(): log = run_ossify(tree, rust_roots(tree)) edits = deactivate_enterprise(tree) + deactivate_enterprise_in_scripts(tree) dangling = remove_dangling_mods(tree) + renames = rename_upstream_names(tree) problems = verify(tree) gates, checks = remaining_hooks(tree) others = third_party(tree) new_others = unlisted(others) + build = None + if not args.no_build_check and not problems: + print('strip: compiling the stripped tree (cargo check)...', file=sys.stderr) + errors, unused = build_check(tree, args.target_dir) + known = known_build_failures() + expected = [e for e in errors if e['file'] in known] + errors = [e for e in errors if e['file'] not in known] + build = {'errors': errors, 'expected': expected, 'unused': unused, + 'known_clean': sorted(known - {e['file'] for e in expected})} + problems += [f'{e["file"]}:{e["line"]}: does not compile: {e["message"]}' for e in errors] report = { 'ref': args.ref, 'commit': commit, @@ -447,7 +538,7 @@ def main(): 'cargo_edits': edits, 'dangling_mods': dangling, 'problems': problems, 'feature_gates': gates, 'edition_checks': checks, 'schema': schema_flags(tree), 'third_party': others, 'third_party_unlisted': new_others, - 'ossify_log': log, + 'renames': renames, 'build': build, 'ossify_log': log, } write_report(args.out, report) print(f'{args.ref} ({commit[:12]}): removed {len(removed_files)} files and '