diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a3707..3d42a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a granted application permissions, a whole Microsoft 365 tenant moves in a plan with no one signing in. The token is renewed before it runs out. A plan's `clientSecret` now reaches Graph as well as EWS. +- `import imap --google-key` signs in to Gmail as a Google Workspace service + account with domain-wide delegation, acting for the user `--auth-user` + names. A whole Workspace domain moves in a plan with one key and no + one's password. The JWT is signed with the key itself (RS256), and each + connection takes a fresh token when the last is close to running out. ## [27.1-rc.5] diff --git a/Cargo.lock b/Cargo.lock index 089cc33..b16ccb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,6 +1327,7 @@ dependencies = [ name = "inbuxa-migrate" version = "27.1.0-rc.5" dependencies = [ + "aws-lc-rs", "blake3", "calcard", "chrono", diff --git a/Cargo.toml b/Cargo.toml index d24791d..f3af99a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ psl = "2.1.226" getrandom = "0.3.4" [dev-dependencies] +aws-lc-rs = { version = "1.18", default-features = false, features = ["aws-lc-sys"] } mockito = "1.7.2" tempfile = "3.13" testcontainers = { version = "0.28", features = ["blocking"] } diff --git a/docs/usage.md b/docs/usage.md index ea89afc..75a7ff9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -79,7 +79,8 @@ Imports one JMAP account, from inbuxa, Stalwart or any other JMAP server. `--obj ``` inbuxa-migrate import imap \ --url imap(s)://host[:port] \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user \ + | --google-key --auth-user ) \ [--include ...] [--exclude ...] [--exclude-special ...] \ [--folder ...] [--subscribed-only] [--noautomap] \ [--include-deleted] [--allow-cleartext] [--compress] \ @@ -99,6 +100,23 @@ arrives: an interrupted import keeps what it fetched, and the next run picks up from there. A message that can't be imported is reported with its folder and UID, and the rest of the folder carries on. +**Gmail, a whole Google Workspace domain.** With `--google-key` the tool +signs in as a Workspace service account acting for the user `--auth-user` +names, so no one's password is needed. In a plan that is +`--url imaps://imap.gmail.com --google-key key.json --auth-user {email}`. + +- In Google Cloud, create a service account and a JSON key for it. +- In the Workspace Admin console, under domain-wide delegation, allow the + service account's client id the scope `https://mail.google.com/`. + IMAP has to be allowed for the users, in the Admin console's Gmail + settings. +- The key file holds the account's private key and reaches every mailbox + in the domain: keep it readable by you alone, and delete the key in the + Cloud console once the move is done. + +Each connection signs in with a token for that user, and takes a new one +when the last is close to running out. + ### CalDAV ``` @@ -624,8 +642,9 @@ account, the state a later run resumes from, and the report. use a login that can act for each account. The example uses Dovecot's master user, `{email}*master`. Use the source's own form where it has one: Exchange impersonation through `--mailbox {email}`, a Graph app - with `--app-only --user {email}`, or an administrator on a JMAP source - with `--account-name {email}`. + with `--app-only --user {email}`, a Google service account with + `--google-key key.json --auth-user {email}`, or an administrator on a + JMAP source with `--account-name {email}`. - **One account at a time, several at once.** `--parallel` (default 4) sets how many accounts run at the same time. Each step runs as its own process. Its output goes to `//.log`, and one account diff --git a/src/cli.rs b/src/cli.rs index f1d417e..ecfa4b4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -235,7 +235,7 @@ struct JmapImportArgs { } #[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer", "google_key"])))] struct ImapImportArgs { #[arg( long, @@ -269,10 +269,17 @@ struct ImapImportArgs { #[arg( long, value_name = "USER", - help = "XOAUTH2 identity (required with --auth-bearer)" + help = "XOAUTH2 identity (required with --auth-bearer and --google-key)" )] auth_user: Option, + #[arg( + long, + value_name = "FILE", + help = "Google Workspace: a service account's JSON key with domain-wide delegation; signs in as --auth-user" + )] + google_key: Option, + #[arg(long, help = "Permit credentials on imap:// without STARTTLS")] allow_cleartext: bool, @@ -1211,12 +1218,30 @@ fn resolve_jmap_import(args: JmapImportArgs) -> Result { fn resolve_imap_import(args: ImapImportArgs) -> Result { let common = common_config(&args.global, args.archive); - let auth = resolve_imap_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - args.auth_user.as_deref(), - )?; + let auth = match &args.google_key { + Some(key) => { + if args.auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let user = args.auth_user.as_deref().ok_or_else(|| { + Error::Usage("--auth-user names the account --google-key signs in as".to_owned()) + })?; + let account = std::sync::Arc::new(crate::google::ServiceAccount::load(key)?); + ImapAuth::Google(std::sync::Arc::new(crate::google::Delegated::new( + account, + user, + crate::google::MAIL_SCOPE, + ))) + } + None => resolve_imap_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + args.auth_user.as_deref(), + )?, + }; if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) { return Err(Error::Usage( "--folder is mutually exclusive with --include/--exclude".to_owned(), @@ -2108,4 +2133,33 @@ mod tests { _ => panic!("expected an app-only exchange-graph import"), } } + + #[test] + fn imap_google_key_signs_in_as_the_user_auth_user_names() { + let _g = lock(); + let parse = |extra: &[&str]| { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "imap", + "--url", + "imaps://imap.gmail.com", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + Cli::try_parse_from(argv) + }; + assert!(parse(&["--google-key", "k.json", "--auth-basic", "a"]).is_err()); + match parse(&["--google-key", "k.json"]).unwrap().resolve() { + Err(Error::Usage(m)) => assert!(m.contains("--auth-user"), "{m}"), + _ => panic!("--auth-user is needed"), + } + match parse(&["--google-key", "/nonexistent/k.json", "--auth-user", "a@x"]) + .unwrap() + .resolve() + { + Err(Error::Usage(m)) => assert!(m.contains("/nonexistent/k.json"), "{m}"), + _ => panic!("a missing key file is a usage error"), + } + } } diff --git a/src/google.rs b/src/google.rs new file mode 100644 index 0000000..8928690 --- /dev/null +++ b/src/google.rs @@ -0,0 +1,219 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Signing in to Google Workspace as a service account with domain-wide +//! delegation. A short JWT, signed with the account's key, names the user +//! to act for, and Google's token endpoint answers with an access token for +//! that user (RFC 7523). One key reaches every mailbox in the domain, so no +//! one hands over a password. + +use std::fmt; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use encodify::base64::URL_SAFE_NO_PAD; +use rustls::SignatureScheme; +use rustls::pki_types::PrivateKeyDer; +use rustls::pki_types::pem::PemObject; +use rustls::sign::SigningKey; +use serde_json::{Value, json}; +use ureq::config::Config; + +use crate::error::Error; +use crate::net::{tls, with_timeouts}; + +/// Full IMAP access, the scope Gmail's XOAUTH2 needs. +pub const MAIL_SCOPE: &str = "https://mail.google.com/"; +const TOKEN_URI: &str = "https://oauth2.googleapis.com/token"; +/// The longest a Google assertion may ask for. +const LIFETIME_SECS: u64 = 3600; +/// A cached token is renewed when it has less than this left. +const MARGIN: Duration = Duration::from_secs(300); + +/// A service account's JSON key, as the Google Cloud console hands it out. +pub struct ServiceAccount { + client_email: String, + token_uri: String, + key: Arc, +} + +impl fmt::Debug for ServiceAccount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ServiceAccount") + .field("client_email", &self.client_email) + .finish_non_exhaustive() + } +} + +impl ServiceAccount { + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?; + Self::parse(&text).map_err(|e| Error::Usage(format!("{}: {e}", path.display()))) + } + + pub fn parse(text: &str) -> Result { + let doc: Value = serde_json::from_str(text).map_err(|e| e.to_string())?; + let field = |name: &str| doc.get(name).and_then(Value::as_str); + if field("type") != Some("service_account") { + return Err("not a service account's JSON key".to_owned()); + } + let client_email = field("client_email") + .ok_or("the key has no client_email")? + .to_owned(); + let pem = field("private_key").ok_or("the key has no private_key")?; + let der = PrivateKeyDer::from_pem_slice(pem.as_bytes()) + .map_err(|e| format!("private_key: {e}"))?; + let key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&der) + .map_err(|e| format!("private_key: {e}"))?; + Ok(Self { + client_email, + token_uri: field("token_uri").unwrap_or(TOKEN_URI).to_owned(), + key, + }) + } + + pub fn client_email(&self) -> &str { + &self.client_email + } + + /// The signed JWT that asks for `scope` as `user`. + pub fn assertion(&self, user: &str, scope: &str, now: u64) -> Result { + let header = json!({"alg": "RS256", "typ": "JWT"}); + let claims = json!({ + "iss": self.client_email, "sub": user, "scope": scope, + "aud": self.token_uri, "iat": now, "exp": now + LIFETIME_SECS, + }); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(header.to_string().as_bytes()), + URL_SAFE_NO_PAD.encode(claims.to_string().as_bytes()) + ); + let signer = self + .key + .choose_scheme(&[SignatureScheme::RSA_PKCS1_SHA256]) + .ok_or("the key can't sign RS256: Google's keys are RSA")?; + let signature = signer + .sign(input.as_bytes()) + .map_err(|e| format!("signing: {e}"))?; + Ok(format!("{input}.{}", URL_SAFE_NO_PAD.encode(&signature))) + } + + /// An access token for `user`, and how long it lasts. + pub fn token_for(&self, user: &str, scope: &str) -> Result<(String, Duration), Error> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let assertion = self.assertion(user, scope, now).map_err(Error::Usage)?; + let body = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer") + .append_pair("assertion", &assertion) + .finish(); + let config: Config = with_timeouts!( + Config::builder() + .http_status_as_error(false) + .tls_config(tls(false)) + ) + .build(); + let mut resp = config + .new_agent() + .post(&self.token_uri) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| Error::Connection(format!("Google token endpoint: {e}")))?; + let status = resp.status().as_u16(); + let reply: Value = resp.body_mut().read_json().unwrap_or(Value::Null); + if let Some(token) = reply.get("access_token").and_then(Value::as_str) + && status == 200 + { + let lasts = reply + .get("expires_in") + .and_then(Value::as_u64) + .unwrap_or(LIFETIME_SECS); + return Ok((token.to_owned(), Duration::from_secs(lasts))); + } + let error = reply.get("error").and_then(Value::as_str).unwrap_or(""); + let detail = reply + .get("error_description") + .and_then(Value::as_str) + .unwrap_or(""); + let hint = if error == "unauthorized_client" { + format!( + "; in the Admin console, allow this service account's client id \ + domain-wide delegation for {scope}" + ) + } else { + String::new() + }; + Err(Error::Connection(format!( + "Google refused a token for {user} as {} (http {status}): {error} {detail}{hint}", + self.client_email + ))) + } +} + +/// A service account acting for one user, with the token it last got, +/// renewed when it's close to running out. Each connection signs in with +/// it, so a run longer than a token's hour takes a new one on reconnect. +pub struct Delegated { + account: Arc, + user: String, + scope: &'static str, + cached: Mutex>, +} + +impl fmt::Debug for Delegated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Delegated") + .field("account", &self.account.client_email) + .field("user", &self.user) + .finish_non_exhaustive() + } +} + +impl Delegated { + pub fn new(account: Arc, user: &str, scope: &'static str) -> Self { + Self { + account, + user: user.to_owned(), + scope, + cached: Mutex::new(None), + } + } + + pub fn user(&self) -> &str { + &self.user + } + + pub fn token(&self) -> Result { + let mut cached = self.cached.lock().unwrap_or_else(|e| e.into_inner()); + if let Some((token, until)) = cached.as_ref() + && Instant::now() + MARGIN < *until + { + return Ok(token.clone()); + } + let (token, lasts) = self.account.token_for(&self.user, self.scope)?; + *cached = Some((token.clone(), Instant::now() + lasts)); + Ok(token) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_key_that_isnt_a_service_accounts_is_refused() { + let e = ServiceAccount::parse(r#"{"type":"authorized_user"}"#).unwrap_err(); + assert!(e.contains("service account"), "{e}"); + let e = ServiceAccount::parse( + r#"{"type":"service_account","client_email":"a@p.iam.gserviceaccount.com","private_key":"nope"}"#, + ) + .unwrap_err(); + assert!(e.contains("private_key"), "{e}"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 7b29d9b..a8cd955 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ pub mod exchange_ews; pub mod exchange_graph; pub mod export_directory; pub mod export_files; +pub mod google; pub mod guide; pub mod imap; pub mod import_directory; diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs index aafb3de..13d24f0 100644 --- a/src/sync/import_imap/coordinator.rs +++ b/src/sync/import_imap/coordinator.rs @@ -208,8 +208,17 @@ pub struct ImapImportConfig { #[derive(Debug, Clone)] pub enum ImapAuth { - Basic { user: String, password: String }, - Bearer { user: String, token: String }, + Basic { + user: String, + password: String, + }, + Bearer { + user: String, + token: String, + }, + /// A Google Workspace service account acting for the user: each sign-in + /// takes a token from it, a new one once the last is nearly spent. + Google(std::sync::Arc), } pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result { @@ -517,6 +526,13 @@ pub(super) fn authenticate_client(client: &mut ImapClient, auth: &ImapAuth) -> R pub(crate) fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result { match auth { + ImapAuth::Google(delegated) => { + let bearer = ImapAuth::Bearer { + user: delegated.user().to_owned(), + token: delegated.token()?, + }; + authenticate(client, &bearer) + } ImapAuth::Basic { user, password } => { if client.has_capability("AUTH=PLAIN") { match client.authenticate_plain(user, password) { diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index 394c167..258c381 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -2308,3 +2308,111 @@ fn append_refused_before_the_go_ahead_sends_nothing_and_keeps_the_session() { } client.noop().expect("session still usable"); } + +/// A service account's JSON key for a new RSA key, with the token +/// endpoint at `token_uri`, and the key's public half. +fn google_key(token_uri: &str) -> (String, Vec) { + use aws_lc_rs::encoding::{AsDer, Pkcs8V1Der}; + use aws_lc_rs::signature::KeyPair as _; + let pair = aws_lc_rs::rsa::KeyPair::generate(aws_lc_rs::rsa::KeySize::Rsa2048).unwrap(); + let der: Pkcs8V1Der<'static> = pair.as_der().unwrap(); + let body = STANDARD.encode(der.as_ref()); + let lines: Vec<&str> = body + .as_bytes() + .chunks(64) + .map(|c| std::str::from_utf8(c).unwrap()) + .collect(); + let pem = format!( + "-----BEGIN PRIVATE KEY-----\n{}\n-----END PRIVATE KEY-----\n", + lines.join("\n") + ); + let key = serde_json::json!({ + "type": "service_account", "client_email": "migrate@proj.iam.gserviceaccount.com", + "private_key": pem, "token_uri": token_uri, + }); + (key.to_string(), pair.public_key().as_ref().to_vec()) +} + +#[test] +fn coordinator_signs_in_to_gmail_as_a_service_account_acting_for_the_user() { + let mut google = mockito::Server::new(); + let token_uri = format!("{}/token", google.url()); + let (key, public) = google_key(&token_uri); + let audience = token_uri.clone(); + let _token = google + .mock("POST", "/token") + .match_body(mockito::Matcher::Regex( + "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer".to_owned(), + )) + .with_body_from_request(move |request| { + let body = String::from_utf8_lossy(request.body().unwrap()).into_owned(); + let assertion = body + .split('&') + .find_map(|p| p.strip_prefix("assertion=")) + .unwrap(); + let (signed, signature) = assertion.rsplit_once('.').unwrap(); + let signature = encodify::base64::URL_SAFE_NO_PAD.decode(signature).unwrap(); + aws_lc_rs::signature::UnparsedPublicKey::new( + &aws_lc_rs::signature::RSA_PKCS1_2048_8192_SHA256, + &public, + ) + .verify(signed.as_bytes(), &signature) + .expect("the assertion is signed with the account's key"); + let claims = signed.split('.').nth(1).unwrap(); + let claims: serde_json::Value = + serde_json::from_slice(&encodify::base64::URL_SAFE_NO_PAD.decode(claims).unwrap()) + .unwrap(); + assert_eq!(claims["iss"], "migrate@proj.iam.gserviceaccount.com"); + assert_eq!(claims["sub"], "alice@example.org"); + assert_eq!(claims["scope"], "https://mail.google.com/"); + assert_eq!(claims["aud"], audience.as_str()); + br#"{"access_token":"ya29.for-alice","expires_in":3599}"#.to_vec() + }) + .expect(1) + .create(); + + const CAPS: &str = "IMAP4rev2 LITERAL+ SASL-IR AUTH=XOAUTH2"; + let control: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> { + conn.write_line("* OK ready")?; + let (tag, _) = conn.read_command()?; + write_capability(conn, CAPS)?; + conn.write_line(&format!("{tag} OK done"))?; + let (tag, cmd) = conn.read_command()?; + let payload = cmd.strip_prefix("AUTHENTICATE XOAUTH2 ").unwrap(); + assert_eq!( + STANDARD.decode(payload).unwrap(), + b"user=alice@example.org\x01auth=Bearer ya29.for-alice\x01\x01" + ); + write_post_auth_caps(conn, &tag, CAPS)?; + let (tag, _) = conn.read_command()?; + write_capability(conn, CAPS)?; + conn.write_line(&format!("{tag} OK done"))?; + let (tag, _) = conn.read_command()?; + conn.write_line("* LIST () \"/\" \"INBOX\"")?; + conn.write_line(&format!("{tag} OK"))?; + let (tag, _) = conn.read_command()?; + conn.write_line(&format!("{tag} OK"))?; + let (tag, _) = conn.read_command()?; + write_select(conn, &tag, 100, 1, 0)?; + let (tag, _) = conn.read_command()?; + conn.write_line("* SEARCH")?; + conn.write_line(&format!("{tag} OK"))?; + drain_until_close(conn); + Ok(()) + }); + let server = MockImap::start_scripts(vec![control, worker_idle_script(CAPS)]); + let account = inbuxa_migrate::google::ServiceAccount::parse(&key).unwrap(); + let delegated = inbuxa_migrate::google::Delegated::new( + std::sync::Arc::new(account), + "alice@example.org", + inbuxa_migrate::google::MAIL_SCOPE, + ); + let archive = tempfile("google_service_account"); + let summary = run_import(&server, "alice@example.org", archive, |c| { + c.auth = ImapAuth::Google(std::sync::Arc::new(delegated)); + }) + .expect("import"); + assert!(!summary.any_failed()); + // Both connections signed in on the one token. + _token.assert(); +}