import imap: Gmail as a Workspace service account (--google-key) #45

Merged
jcoffey-dev merged 1 commits from feat/google-service-account into main 2026-10-30 21:43:13 +00:00
9 changed files with 437 additions and 13 deletions

No files matched your search

+5
View File
@@ -13,6 +13,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
granted application permissions, a whole Microsoft 365 tenant moves in a
plan with no one signing in. The token is renewed before it runs out. A
plan's `clientSecret` now reaches Graph as well as EWS.
- `import imap --google-key` signs in to Gmail as a Google Workspace service
account with domain-wide delegation, acting for the user `--auth-user`
names. A whole Workspace domain moves in a plan with one key and no
one's password. The JWT is signed with the key itself (RS256), and each
connection takes a fresh token when the last is close to running out.
## [27.1-rc.5]
Generated
+1
View File
@@ -1327,6 +1327,7 @@ dependencies = [
name = "inbuxa-migrate"
version = "27.1.0-rc.5"
dependencies = [
"aws-lc-rs",
"blake3",
"calcard",
"chrono",
+1
View File
@@ -44,6 +44,7 @@ psl = "2.1.226"
getrandom = "0.3.4"
[dev-dependencies]
aws-lc-rs = { version = "1.18", default-features = false, features = ["aws-lc-sys"] }
mockito = "1.7.2"
tempfile = "3.13"
testcontainers = { version = "0.28", features = ["blocking"] }
+22 -3
View File
@@ -79,7 +79,8 @@ Imports one JMAP account, from inbuxa, Stalwart or any other JMAP server. `--obj
```
inbuxa-migrate import imap \
--url imap(s)://host[:port] \
(--auth-basic <USER> [--auth-password <PASS>] | --auth-bearer [TOKEN] --auth-user <USER>) \
(--auth-basic <USER> [--auth-password <PASS>] | --auth-bearer [TOKEN] --auth-user <USER> \
| --google-key <FILE> --auth-user <USER>) \
[--include <REGEX>...] [--exclude <REGEX>...] [--exclude-special <ROLE>...] \
[--folder <NAME>...] [--subscribed-only] [--noautomap] \
[--include-deleted] [--allow-cleartext] [--compress] \
@@ -99,6 +100,23 @@ arrives: an interrupted import keeps what it fetched, and the next run picks
up from there. A message that can't be imported is reported with its folder
and UID, and the rest of the folder carries on.
**Gmail, a whole Google Workspace domain.** With `--google-key` the tool
signs in as a Workspace service account acting for the user `--auth-user`
names, so no one's password is needed. In a plan that is
`--url imaps://imap.gmail.com --google-key key.json --auth-user {email}`.
- In Google Cloud, create a service account and a JSON key for it.
- In the Workspace Admin console, under domain-wide delegation, allow the
service account's client id the scope `https://mail.google.com/`.
IMAP has to be allowed for the users, in the Admin console's Gmail
settings.
- The key file holds the account's private key and reaches every mailbox
in the domain: keep it readable by you alone, and delete the key in the
Cloud console once the move is done.
Each connection signs in with a token for that user, and takes a new one
when the last is close to running out.
### CalDAV
```
@@ -624,8 +642,9 @@ account, the state a later run resumes from, and the report.
use a login that can act for each account. The example uses Dovecot's
master user, `{email}*master`. Use the source's own form where it has
one: Exchange impersonation through `--mailbox {email}`, a Graph app
with `--app-only --user {email}`, or an administrator on a JMAP source
with `--account-name {email}`.
with `--app-only --user {email}`, a Google service account with
`--google-key key.json --auth-user {email}`, or an administrator on a
JMAP source with `--account-name {email}`.
- **One account at a time, several at once.** `--parallel` (default 4)
sets how many accounts run at the same time. Each step runs as its own
process. Its output goes to `<DIR>/<email>/<step>.log`, and one account
+62 -8
View File
@@ -235,7 +235,7 @@ struct JmapImportArgs {
}
#[derive(Args)]
#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))]
#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer", "google_key"])))]
struct ImapImportArgs {
#[arg(
long,
@@ -269,10 +269,17 @@ struct ImapImportArgs {
#[arg(
long,
value_name = "USER",
help = "XOAUTH2 identity (required with --auth-bearer)"
help = "XOAUTH2 identity (required with --auth-bearer and --google-key)"
)]
auth_user: Option<String>,
#[arg(
long,
value_name = "FILE",
help = "Google Workspace: a service account's JSON key with domain-wide delegation; signs in as --auth-user"
)]
google_key: Option<PathBuf>,
#[arg(long, help = "Permit credentials on imap:// without STARTTLS")]
allow_cleartext: bool,
@@ -1211,12 +1218,30 @@ fn resolve_jmap_import(args: JmapImportArgs) -> Result<Action, Error> {
fn resolve_imap_import(args: ImapImportArgs) -> Result<Action, Error> {
let common = common_config(&args.global, args.archive);
let auth = resolve_imap_auth(
args.auth_basic.as_deref(),
args.auth_password.as_deref(),
args.auth_bearer.as_ref(),
args.auth_user.as_deref(),
)?;
let auth = match &args.google_key {
Some(key) => {
if args.auth_password.is_some() {
return Err(Error::Usage(
"--auth-password is only valid together with --auth-basic".to_owned(),
));
}
let user = args.auth_user.as_deref().ok_or_else(|| {
Error::Usage("--auth-user names the account --google-key signs in as".to_owned())
})?;
let account = std::sync::Arc::new(crate::google::ServiceAccount::load(key)?);
ImapAuth::Google(std::sync::Arc::new(crate::google::Delegated::new(
account,
user,
crate::google::MAIL_SCOPE,
)))
}
None => resolve_imap_auth(
args.auth_basic.as_deref(),
args.auth_password.as_deref(),
args.auth_bearer.as_ref(),
args.auth_user.as_deref(),
)?,
};
if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) {
return Err(Error::Usage(
"--folder is mutually exclusive with --include/--exclude".to_owned(),
@@ -2108,4 +2133,33 @@ mod tests {
_ => panic!("expected an app-only exchange-graph import"),
}
}
#[test]
fn imap_google_key_signs_in_as_the_user_auth_user_names() {
let _g = lock();
let parse = |extra: &[&str]| {
let mut argv = vec![
"inbuxa-migrate",
"import",
"imap",
"--url",
"imaps://imap.gmail.com",
];
argv.extend_from_slice(extra);
argv.push("archive.sqlite");
Cli::try_parse_from(argv)
};
assert!(parse(&["--google-key", "k.json", "--auth-basic", "a"]).is_err());
match parse(&["--google-key", "k.json"]).unwrap().resolve() {
Err(Error::Usage(m)) => assert!(m.contains("--auth-user"), "{m}"),
_ => panic!("--auth-user is needed"),
}
match parse(&["--google-key", "/nonexistent/k.json", "--auth-user", "a@x"])
.unwrap()
.resolve()
{
Err(Error::Usage(m)) => assert!(m.contains("/nonexistent/k.json"), "{m}"),
_ => panic!("a missing key file is a usage error"),
}
}
}
+219
View File
@@ -0,0 +1,219 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! Signing in to Google Workspace as a service account with domain-wide
//! delegation. A short JWT, signed with the account's key, names the user
//! to act for, and Google's token endpoint answers with an access token for
//! that user (RFC 7523). One key reaches every mailbox in the domain, so no
//! one hands over a password.
use std::fmt;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use encodify::base64::URL_SAFE_NO_PAD;
use rustls::SignatureScheme;
use rustls::pki_types::PrivateKeyDer;
use rustls::pki_types::pem::PemObject;
use rustls::sign::SigningKey;
use serde_json::{Value, json};
use ureq::config::Config;
use crate::error::Error;
use crate::net::{tls, with_timeouts};
/// Full IMAP access, the scope Gmail's XOAUTH2 needs.
pub const MAIL_SCOPE: &str = "https://mail.google.com/";
const TOKEN_URI: &str = "https://oauth2.googleapis.com/token";
/// The longest a Google assertion may ask for.
const LIFETIME_SECS: u64 = 3600;
/// A cached token is renewed when it has less than this left.
const MARGIN: Duration = Duration::from_secs(300);
/// A service account's JSON key, as the Google Cloud console hands it out.
pub struct ServiceAccount {
client_email: String,
token_uri: String,
key: Arc<dyn SigningKey>,
}
impl fmt::Debug for ServiceAccount {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ServiceAccount")
.field("client_email", &self.client_email)
.finish_non_exhaustive()
}
}
impl ServiceAccount {
pub fn load(path: &Path) -> Result<Self, Error> {
let text = std::fs::read_to_string(path)
.map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?;
Self::parse(&text).map_err(|e| Error::Usage(format!("{}: {e}", path.display())))
}
pub fn parse(text: &str) -> Result<Self, String> {
let doc: Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
let field = |name: &str| doc.get(name).and_then(Value::as_str);
if field("type") != Some("service_account") {
return Err("not a service account's JSON key".to_owned());
}
let client_email = field("client_email")
.ok_or("the key has no client_email")?
.to_owned();
let pem = field("private_key").ok_or("the key has no private_key")?;
let der = PrivateKeyDer::from_pem_slice(pem.as_bytes())
.map_err(|e| format!("private_key: {e}"))?;
let key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&der)
.map_err(|e| format!("private_key: {e}"))?;
Ok(Self {
client_email,
token_uri: field("token_uri").unwrap_or(TOKEN_URI).to_owned(),
key,
})
}
pub fn client_email(&self) -> &str {
&self.client_email
}
/// The signed JWT that asks for `scope` as `user`.
pub fn assertion(&self, user: &str, scope: &str, now: u64) -> Result<String, String> {
let header = json!({"alg": "RS256", "typ": "JWT"});
let claims = json!({
"iss": self.client_email, "sub": user, "scope": scope,
"aud": self.token_uri, "iat": now, "exp": now + LIFETIME_SECS,
});
let input = format!(
"{}.{}",
URL_SAFE_NO_PAD.encode(header.to_string().as_bytes()),
URL_SAFE_NO_PAD.encode(claims.to_string().as_bytes())
);
let signer = self
.key
.choose_scheme(&[SignatureScheme::RSA_PKCS1_SHA256])
.ok_or("the key can't sign RS256: Google's keys are RSA")?;
let signature = signer
.sign(input.as_bytes())
.map_err(|e| format!("signing: {e}"))?;
Ok(format!("{input}.{}", URL_SAFE_NO_PAD.encode(&signature)))
}
/// An access token for `user`, and how long it lasts.
pub fn token_for(&self, user: &str, scope: &str) -> Result<(String, Duration), Error> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
let assertion = self.assertion(user, scope, now).map_err(Error::Usage)?;
let body = url::form_urlencoded::Serializer::new(String::new())
.append_pair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
.append_pair("assertion", &assertion)
.finish();
let config: Config = with_timeouts!(
Config::builder()
.http_status_as_error(false)
.tls_config(tls(false))
)
.build();
let mut resp = config
.new_agent()
.post(&self.token_uri)
.header("Content-Type", "application/x-www-form-urlencoded")
.send(body.as_bytes())
.map_err(|e| Error::Connection(format!("Google token endpoint: {e}")))?;
let status = resp.status().as_u16();
let reply: Value = resp.body_mut().read_json().unwrap_or(Value::Null);
if let Some(token) = reply.get("access_token").and_then(Value::as_str)
&& status == 200
{
let lasts = reply
.get("expires_in")
.and_then(Value::as_u64)
.unwrap_or(LIFETIME_SECS);
return Ok((token.to_owned(), Duration::from_secs(lasts)));
}
let error = reply.get("error").and_then(Value::as_str).unwrap_or("");
let detail = reply
.get("error_description")
.and_then(Value::as_str)
.unwrap_or("");
let hint = if error == "unauthorized_client" {
format!(
"; in the Admin console, allow this service account's client id \
domain-wide delegation for {scope}"
)
} else {
String::new()
};
Err(Error::Connection(format!(
"Google refused a token for {user} as {} (http {status}): {error} {detail}{hint}",
self.client_email
)))
}
}
/// A service account acting for one user, with the token it last got,
/// renewed when it's close to running out. Each connection signs in with
/// it, so a run longer than a token's hour takes a new one on reconnect.
pub struct Delegated {
account: Arc<ServiceAccount>,
user: String,
scope: &'static str,
cached: Mutex<Option<(String, Instant)>>,
}
impl fmt::Debug for Delegated {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Delegated")
.field("account", &self.account.client_email)
.field("user", &self.user)
.finish_non_exhaustive()
}
}
impl Delegated {
pub fn new(account: Arc<ServiceAccount>, user: &str, scope: &'static str) -> Self {
Self {
account,
user: user.to_owned(),
scope,
cached: Mutex::new(None),
}
}
pub fn user(&self) -> &str {
&self.user
}
pub fn token(&self) -> Result<String, Error> {
let mut cached = self.cached.lock().unwrap_or_else(|e| e.into_inner());
if let Some((token, until)) = cached.as_ref()
&& Instant::now() + MARGIN < *until
{
return Ok(token.clone());
}
let (token, lasts) = self.account.token_for(&self.user, self.scope)?;
*cached = Some((token.clone(), Instant::now() + lasts));
Ok(token)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_key_that_isnt_a_service_accounts_is_refused() {
let e = ServiceAccount::parse(r#"{"type":"authorized_user"}"#).unwrap_err();
assert!(e.contains("service account"), "{e}");
let e = ServiceAccount::parse(
r#"{"type":"service_account","client_email":"[email protected]","private_key":"nope"}"#,
)
.unwrap_err();
assert!(e.contains("private_key"), "{e}");
}
}
+1
View File
@@ -14,6 +14,7 @@ pub mod exchange_ews;
pub mod exchange_graph;
pub mod export_directory;
pub mod export_files;
pub mod google;
pub mod guide;
pub mod imap;
pub mod import_directory;
+18 -2
View File
@@ -208,8 +208,17 @@ pub struct ImapImportConfig {
#[derive(Debug, Clone)]
pub enum ImapAuth {
Basic { user: String, password: String },
Bearer { user: String, token: String },
Basic {
user: String,
password: String,
},
Bearer {
user: String,
token: String,
},
/// A Google Workspace service account acting for the user: each sign-in
/// takes a token from it, a new one once the last is nearly spent.
Google(std::sync::Arc<crate::google::Delegated>),
}
pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> {
@@ -517,6 +526,13 @@ pub(super) fn authenticate_client(client: &mut ImapClient, auth: &ImapAuth) -> R
pub(crate) fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result<String, Error> {
match auth {
ImapAuth::Google(delegated) => {
let bearer = ImapAuth::Bearer {
user: delegated.user().to_owned(),
token: delegated.token()?,
};
authenticate(client, &bearer)
}
ImapAuth::Basic { user, password } => {
if client.has_capability("AUTH=PLAIN") {
match client.authenticate_plain(user, password) {
+108
View File
@@ -2308,3 +2308,111 @@ fn append_refused_before_the_go_ahead_sends_nothing_and_keeps_the_session() {
}
client.noop().expect("session still usable");
}
/// A service account's JSON key for a new RSA key, with the token
/// endpoint at `token_uri`, and the key's public half.
fn google_key(token_uri: &str) -> (String, Vec<u8>) {
use aws_lc_rs::encoding::{AsDer, Pkcs8V1Der};
use aws_lc_rs::signature::KeyPair as _;
let pair = aws_lc_rs::rsa::KeyPair::generate(aws_lc_rs::rsa::KeySize::Rsa2048).unwrap();
let der: Pkcs8V1Der<'static> = pair.as_der().unwrap();
let body = STANDARD.encode(der.as_ref());
let lines: Vec<&str> = body
.as_bytes()
.chunks(64)
.map(|c| std::str::from_utf8(c).unwrap())
.collect();
let pem = format!(
"-----BEGIN PRIVATE KEY-----\n{}\n-----END PRIVATE KEY-----\n",
lines.join("\n")
);
let key = serde_json::json!({
"type": "service_account", "client_email": "[email protected]",
"private_key": pem, "token_uri": token_uri,
});
(key.to_string(), pair.public_key().as_ref().to_vec())
}
#[test]
fn coordinator_signs_in_to_gmail_as_a_service_account_acting_for_the_user() {
let mut google = mockito::Server::new();
let token_uri = format!("{}/token", google.url());
let (key, public) = google_key(&token_uri);
let audience = token_uri.clone();
let _token = google
.mock("POST", "/token")
.match_body(mockito::Matcher::Regex(
"grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer".to_owned(),
))
.with_body_from_request(move |request| {
let body = String::from_utf8_lossy(request.body().unwrap()).into_owned();
let assertion = body
.split('&')
.find_map(|p| p.strip_prefix("assertion="))
.unwrap();
let (signed, signature) = assertion.rsplit_once('.').unwrap();
let signature = encodify::base64::URL_SAFE_NO_PAD.decode(signature).unwrap();
aws_lc_rs::signature::UnparsedPublicKey::new(
&aws_lc_rs::signature::RSA_PKCS1_2048_8192_SHA256,
&public,
)
.verify(signed.as_bytes(), &signature)
.expect("the assertion is signed with the account's key");
let claims = signed.split('.').nth(1).unwrap();
let claims: serde_json::Value =
serde_json::from_slice(&encodify::base64::URL_SAFE_NO_PAD.decode(claims).unwrap())
.unwrap();
assert_eq!(claims["iss"], "[email protected]");
assert_eq!(claims["sub"], "[email protected]");
assert_eq!(claims["scope"], "https://mail.google.com/");
assert_eq!(claims["aud"], audience.as_str());
br#"{"access_token":"ya29.for-alice","expires_in":3599}"#.to_vec()
})
.expect(1)
.create();
const CAPS: &str = "IMAP4rev2 LITERAL+ SASL-IR AUTH=XOAUTH2";
let control: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
conn.write_line("* OK ready")?;
let (tag, _) = conn.read_command()?;
write_capability(conn, CAPS)?;
conn.write_line(&format!("{tag} OK done"))?;
let (tag, cmd) = conn.read_command()?;
let payload = cmd.strip_prefix("AUTHENTICATE XOAUTH2 ").unwrap();
assert_eq!(
STANDARD.decode(payload).unwrap(),
b"[email protected]\x01auth=Bearer ya29.for-alice\x01\x01"
);
write_post_auth_caps(conn, &tag, CAPS)?;
let (tag, _) = conn.read_command()?;
write_capability(conn, CAPS)?;
conn.write_line(&format!("{tag} OK done"))?;
let (tag, _) = conn.read_command()?;
conn.write_line("* LIST () \"/\" \"INBOX\"")?;
conn.write_line(&format!("{tag} OK"))?;
let (tag, _) = conn.read_command()?;
conn.write_line(&format!("{tag} OK"))?;
let (tag, _) = conn.read_command()?;
write_select(conn, &tag, 100, 1, 0)?;
let (tag, _) = conn.read_command()?;
conn.write_line("* SEARCH")?;
conn.write_line(&format!("{tag} OK"))?;
drain_until_close(conn);
Ok(())
});
let server = MockImap::start_scripts(vec![control, worker_idle_script(CAPS)]);
let account = inbuxa_migrate::google::ServiceAccount::parse(&key).unwrap();
let delegated = inbuxa_migrate::google::Delegated::new(
std::sync::Arc::new(account),
"[email protected]",
inbuxa_migrate::google::MAIL_SCOPE,
);
let archive = tempfile("google_service_account");
let summary = run_import(&server, "[email protected]", archive, |c| {
c.auth = ImapAuth::Google(std::sync::Arc::new(delegated));
})
.expect("import");
assert!(!summary.any_failed());
// Both connections signed in on the one token.
_token.assert();
}