ADHOC
/
replicant-packages_apps_Email
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Imap2 additions 

* Implement first-pass Imap2 server-side search
* Improve number parsing performance
* Better handle the BodyThread (loading message bodies)

Change-Id: I0ccd7377c80a0553b086d5204b211067896a2f49
This commit is contained in:
Marc Blank 2012-07-25 09:00:45 -07:00
parent 9c89f85b07
commit 0b6b83c6f9
6 changed files with 375 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
emailsync/src/com/android/emailsync/SyncManager.java
View File

@ -1270,13 +1270,13 @@ public abstract class SyncManager extends Service implements Runnable {
}
}
private void setMailboxSyncStatus(long id, int status) {
public void setMailboxSyncStatus(long id, int status) {
ContentValues values = new ContentValues();
values.put(Mailbox.UI_SYNC_STATUS, status);
mResolver.update(ContentUris.withAppendedId(Mailbox.CONTENT_URI, id), values, null, null);
}
private void setMailboxLastSyncResult(long id, int result) {
public void setMailboxLastSyncResult(long id, int result) {
ContentValues values = new ContentValues();
values.put(Mailbox.UI_LAST_SYNC_RESULT, result);
mResolver.update(ContentUris.withAppendedId(Mailbox.CONTENT_URI, id), values, null, null);
@ -2008,6 +2008,7 @@ public abstract class SyncManager extends Service implements Runnable {
static public void sendMessageRequest(Request req) {
SyncManager ssm = INSTANCE;
if (ssm == null) return;
Message msg = Message.restoreMessageWithId(ssm, req.mMessageId);
if (msg == null) return;
long mailboxId = msg.mMailboxKey;
@ -2029,7 +2030,12 @@ public abstract class SyncManager extends Service implements Runnable {
}
}
}
sendRequest(mailboxId, req);
}
static public void sendRequest(long mailboxId, Request req) {
SyncManager ssm = INSTANCE;
if (ssm == null) return;
AbstractSyncService service = ssm.mServiceMap.get(mailboxId);
if (service == null) {
startManualSync(mailboxId, SYNC_SERVICE_PART_REQUEST, req);

21
imap2/src/com/android/imap2/Imap2SyncManager.java
View File

@ -45,7 +45,9 @@ import com.android.emailcommon.service.SearchParams;
import com.android.emailsync.AbstractSyncService;
import com.android.emailsync.PartRequest;
import com.android.emailsync.SyncManager;
import com.android.mail.providers.UIProvider;
import com.android.mail.providers.UIProvider.AccountCapabilities;
import com.android.mail.providers.UIProvider.LastSyncResult;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
@ -183,8 +185,23 @@ public class Imap2SyncManager extends SyncManager {
@Override
public int searchMessages(long accountId, SearchParams params, long destMailboxId)
throws RemoteException {
// TODO Auto-generated method stub
return 0;
SyncManager ssm = INSTANCE;
if (ssm == null) return 0;
Mailbox mailbox = Mailbox.restoreMailboxWithId(ssm, params.mMailboxId);
Imap2SyncService svc = new Imap2SyncService(ssm, mailbox);
setMailboxSyncStatus(destMailboxId, UIProvider.SyncStatus.USER_QUERY);
boolean ioError = false;
try {
return svc.searchMailbox(ssm, accountId, params, destMailboxId);
} catch (IOException e) {
ioError = true;
return 0;
} finally {
// Report ioError status back
setMailboxLastSyncResult(destMailboxId,
ioError ? LastSyncResult.CONNECTION_ERROR : LastSyncResult.SUCCESS);
setMailboxSyncStatus(destMailboxId, UIProvider.SyncStatus.NO_SYNC);
}
}
@Override

341
imap2/src/com/android/imap2/Imap2SyncService.java
View File

@ -27,7 +27,9 @@ import android.net.TrafficStats;
import android.net.Uri;
import android.os.Bundle;
import android.os.RemoteException;
import android.util.Log;
import com.android.emailcommon.Logging;
import com.android.emailcommon.TrafficFlags;
import com.android.emailcommon.internet.MimeUtility;
import com.android.emailcommon.internet.Rfc822Output;
@ -48,6 +50,7 @@ import com.android.emailcommon.provider.Mailbox;
import com.android.emailcommon.provider.MailboxUtilities;
import com.android.emailcommon.service.EmailServiceProxy;
import com.android.emailcommon.service.EmailServiceStatus;
import com.android.emailcommon.service.SearchParams;
import com.android.emailcommon.service.SyncWindow;
import com.android.emailcommon.utility.CountingOutputStream;
import com.android.emailcommon.utility.EOLConvertingOutputStream;
@ -61,6 +64,7 @@ import com.android.emailsync.SyncManager;
import com.android.imap2.smtp.SmtpSender;
import com.android.mail.providers.UIProvider;
import com.beetstra.jutf7.CharsetProvider;
import com.google.common.annotations.VisibleForTesting;
import java.io.BufferedWriter;
import java.io.IOException;
@ -77,7 +81,10 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -115,6 +122,7 @@ public class Imap2SyncService extends AbstractSyncService {
private static final int SOCKET_CONNECT_TIMEOUT = 10*SECONDS;
private static final int SOCKET_TIMEOUT = 20*SECONDS;
private static final int SEARCH_TIMEOUT = 60*SECONDS;
private static final int AUTOMATIC_SYNC_WINDOW_MAX_MESSAGES = 250;
private static final int AUTOMATIC_SYNC_WINDOW_LARGE_MAILBOX = 1000;
@ -295,22 +303,28 @@ public class Imap2SyncService extends AbstractSyncService {
mStop = true;
}
public String writeCommand (Writer out, String cmd) {
public String writeCommand (Writer out, String cmd) throws IOException {
Integer t = mWriterTag++;
String tag = "@@a" + t + ' ';
if (!cmd.startsWith("login")) {
userLog(tag + cmd);
}
out.write(tag);
out.write(cmd);
out.write("\r\n");
out.flush();
return tag;
}
private void writeContinuation(Writer out, String cmd) {
try {
Integer t = mWriterTag++;
String tag = "@@a" + t + ' ';
out.write(tag);
out.write(cmd);
out.write("\r\n");
out.flush();
if (!cmd.startsWith("login")) {
userLog(tag + cmd);
}
return tag;
userLog(cmd);
} catch (IOException e) {
userLog("IOException in writeCommand");
}
return null;
}
private long readLong (String str, int idx) {
@ -340,7 +354,7 @@ public class Imap2SyncService extends AbstractSyncService {
}
} catch (NumberFormatException e) {
}
else if (mMailbox != null && mMailbox.mSyncKey == null || mMailbox.mSyncKey == "0") {
else if (mMailbox != null && (mMailbox.mSyncKey == null || mMailbox.mSyncKey == "0")) {
str = str.toLowerCase();
int idx = str.indexOf("uidvalidity");
if (idx > 0) {
@ -389,6 +403,9 @@ public class Imap2SyncService extends AbstractSyncService {
readUntagged(str);
} else
readUntagged(str);
} else if (str.charAt(0) == '+') {
mImapResult = str;
return str;
} else if (!mImapResponse.isEmpty()) {
// Continuation with string literal, perhaps?
int off = mImapResponse.size() - 1;
@ -432,7 +449,7 @@ public class Imap2SyncService extends AbstractSyncService {
return Address.toFriendly(Address.unpack(msg.mFrom));
}
private Message createMessage (String str) {
private Message createMessage (String str, long mailboxId) {
Parser p = new Parser(str, str.indexOf('(') + 1);
Date date = null;
String subject = null;
@ -444,7 +461,7 @@ public class Imap2SyncService extends AbstractSyncService {
boolean bodystructure = false;
Message msg = new Message();
msg.mMailboxKey = mMailboxId;
msg.mMailboxKey = mailboxId;
try {
while (true) {
@ -504,6 +521,11 @@ public class Imap2SyncService extends AbstractSyncService {
msg.mFlagRead = true;
msg.mTimeStamp = ((date != null) ? date : new Date()).getTime();
msg.mServerId = Long.toString(uid);
// If we're not storing to the same mailbox (search), save away our mailbox name
if (mailboxId != mMailboxId) {
msg.mProtocolSearchInfo = mMailboxName;
}
return msg;
}
@ -807,7 +829,7 @@ public class Imap2SyncService extends AbstractSyncService {
// First, generate the appropriate String
while (!updates.isEmpty()) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 20 && !updates.empty(); i++) {
for (int i = 0; i < HEADER_BATCH_COUNT && !updates.empty(); i++) {
Integer update = updates.pop();
if (i != 0) {
sb.append(',');
@ -926,7 +948,7 @@ public class Imap2SyncService extends AbstractSyncService {
}
}
private Thread mBodyThread;
private BodyThread mBodyThread;
private Connection mConnection;
private void parseBodystructure (Message msg, Parser p, String level, int cnt,
@ -1103,9 +1125,40 @@ public class Imap2SyncService extends AbstractSyncService {
}
}
/**
* Class that loads message bodies in its own thread
*/
private class BodyThread extends Thread {
final Connection mConnection;
final Cursor mCursor;
BodyThread(Connection conn, Cursor cursor) {
super();
mConnection = conn;
mCursor = cursor;
}
public void run() {
try {
fetchMessageData(mConnection, mCursor);
} catch (IOException e) {
userLog("IOException in body thread; closing...");
} finally {
mConnection.close();
mBodyThread = null;
}
}
void close() {
mConnection.close();
}
}
private void fetchMessageData () throws IOException {
// If we're already loading messages on another thread, there's nothing to do
if (mBodyThread != null) return;
if (mBodyThread != null) {
return;
}
HostAuth hostAuth =
HostAuth.restoreHostAuthWithId(mContext, mAccount.mHostAuthKeyRecv);
if (hostAuth == null) return;
@ -1122,19 +1175,9 @@ public class Imap2SyncService extends AbstractSyncService {
if (cnt > 1) {
final Connection conn = connectAndLogin(hostAuth, "body");
if (conn.status == EXIT_DONE) {
mBodyThread =
new Thread(new Runnable() {
@Override
public void run() {
try {
fetchMessageData(conn, unloaded);
conn.socket.close();
} catch (IOException e) {
} finally {
mBodyThread = null;
}
}});
mBodyThread = new BodyThread(conn, unloaded);
mBodyThread.start();
userLog("***** Starting mBodyThread " + mBodyThread.getId());
} else {
fetchMessageData(mConnection, unloaded);
}
@ -1524,11 +1567,22 @@ public class Imap2SyncService extends AbstractSyncService {
return tokens;
}
/**
* Convenience class to hold state for a single IMAP connection
*/
public static class Connection {
Socket socket;
int status;
ImapInputStream reader;
BufferedWriter writer;
void close() {
try {
socket.close();
} catch (IOException e) {
// It's all good
}
}
}
private String mUserAgent;
@ -1702,7 +1756,7 @@ public class Imap2SyncService extends AbstractSyncService {
// We might have left IDLE due to an exception
if (mSocket != null) {
// Reset the standard timeout
mSocket.setSoTimeout(20 * 1000);
mSocket.setSoTimeout(SOCKET_TIMEOUT);
}
mIsIdle = false;
mThread.setName(mMailboxName + "[" + mAccount.mDisplayName + "]");
@ -1863,6 +1917,40 @@ public class Imap2SyncService extends AbstractSyncService {
}
}
private void loadMessages(ArrayList<Integer> loadList, long mailboxId) throws IOException {
int idx= 1;
boolean loadedSome = false;
int cnt = loadList.size();
while (idx <= cnt) {
ArrayList<Message> tmsgList = new ArrayList<Message> ();
int tcnt = 0;
StringBuilder tsb = new StringBuilder("uid fetch ");
for (tcnt = 0; tcnt < HEADER_BATCH_COUNT && idx <= cnt; tcnt++, idx++) {
// Load most recent first
if (tcnt > 0)
tsb.append(',');
tsb.append(loadList.get(cnt - idx));
}
tsb.append(" (uid internaldate flags envelope bodystructure)");
String tag = writeCommand(mWriter, tsb.toString());
if (readResponse(mReader, tag, "FETCH").equals(IMAP_OK)) {
// Create message and store
for (int j = 0; j < tcnt; j++) {
Message msg = createMessage(mImapResponse.get(j), mailboxId);
tmsgList.add(msg);
}
saveNewMessages(tmsgList);
}
fetchMessageData();
loadedSome = true;
}
// TODO: Use loader to watch for changes on unloaded body cursor
if (!loadedSome) {
fetchMessageData();
}
}
private void sync () throws IOException {
mThread = Thread.currentThread();
@ -1920,7 +2008,6 @@ public class Imap2SyncService extends AbstractSyncService {
Date date = new Date(System.currentTimeMillis() - (days*DAYS));
String since = IMAP_DATE_FORMAT.format(date);
String tag;
int[] serverList = getServerIds(since);
if (serverList == null) {
// Do backoff; hope it works next time. Should never happen
@ -1936,39 +2023,9 @@ public class Imap2SyncService extends AbstractSyncService {
ArrayList<Integer> deleteList = r.delete;
serverList = null;
deviceList = null;
int cnt = loadList.size();
// We load message headers in batches
int idx= 1;
boolean loadedSome = false;
while (idx <= cnt) {
ArrayList<Message> tmsgList = new ArrayList<Message> ();
int tcnt = 0;
StringBuilder tsb = new StringBuilder("uid fetch ");
for (tcnt = 0; tcnt < HEADER_BATCH_COUNT && idx <= cnt; tcnt++, idx++) {
// Load most recent first
if (tcnt > 0)
tsb.append(',');
tsb.append(loadList.get(cnt - idx));
}
tsb.append(" (uid internaldate flags envelope bodystructure)");
tag = writeCommand(mWriter, tsb.toString());
if (readResponse(mReader, tag, "FETCH").equals(IMAP_OK)) {
// Create message and store
for (int j = 0; j < tcnt; j++) {
Message msg = createMessage(mImapResponse.get(j));
tmsgList.add(msg);
}
saveNewMessages(tmsgList);
}
fetchMessageData();
loadedSome = true;
}
// TODO: Use loader to watch for changes on unloaded body cursor
if (!loadedSome) {
fetchMessageData();
}
loadMessages(loadList, mMailboxId);
// Reflect server deletions on device; do them all at once
processServerDeletes(deleteList);
@ -1998,11 +2055,11 @@ public class Imap2SyncService extends AbstractSyncService {
}
} finally {
if (mSocket != null) {
if (mConnection != null) {
try {
// Try to logout
readResponse(mReader, writeCommand(mWriter, "logout"));
mSocket.close();
mConnection.close();
} catch (IOException e) {
// We're leaving anyway
}
@ -2107,7 +2164,7 @@ public class Imap2SyncService extends AbstractSyncService {
userLog("Caught IOException: ", (message == null) ? "No message" : message);
mExitStatus = EXIT_IO_ERROR;
} catch (Exception e) {
userLog("Uncaught exception in EasSyncService", e);
userLog("Uncaught exception in Imap2SyncService", e);
} finally {
int status;
Imap2SyncManager.done(this);
@ -2156,6 +2213,11 @@ public class Imap2SyncService extends AbstractSyncService {
// Don't care if this fails
}
// Make sure we close our body thread (if any)
if (mBodyThread != null) {
mBodyThread.close();
}
// Make sure ExchangeService knows about this
Imap2SyncManager.kick("sync finished");
}
@ -2221,4 +2283,157 @@ public class Imap2SyncService extends AbstractSyncService {
"Certificate hostname not useable for server: " + hostname);
}
}
/**
* Cache search results by account; this allows for "load more" support without having to
* redo the search (which can be quite slow).
*/
private static final HashMap<Long, Integer[]> sSearchResults = new HashMap<Long, Integer[]>();
@VisibleForTesting
protected static boolean isAsciiString(String str) {
int len = str.length();
for (int i = 0; i < len; i++) {
char c = str.charAt(i);
if (c >= 128) return false;
}
return true;
}
/**
* Wrapper for a search result with possible exception (to be sent back to the UI)
*/
private static class SearchResult {
Integer[] uids;
Exception exception;
SearchResult(Integer[] _uids, Exception _exception) {
uids = _uids;
exception = _exception;
}
}
private SearchResult getSearchResults(SearchParams searchParams) {
String filter = searchParams.mFilter;
// All servers MUST accept US-ASCII, so we'll send this as the CHARSET unless we're really
// dealing with a string that contains non-ascii characters
String charset = "US-ASCII";
if (!isAsciiString(filter)) {
charset = "UTF-8";
}
List<String> commands = new ArrayList<String>();
// This is the length of the string in octets (bytes), formatted as a string literal {n}
String octetLength = "{" + filter.getBytes().length + "}";
// Break the command up into pieces ending with the string literal length
commands.add("UID SEARCH CHARSET " + charset + " OR FROM " + octetLength);
commands.add(filter + " (OR TO " + octetLength);
commands.add(filter + " (OR CC " + octetLength);
commands.add(filter + " (OR SUBJECT " + octetLength);
commands.add(filter + " BODY " + octetLength);
commands.add(filter + ")))");
Exception exception = null;
try {
int len = commands.size();
String tag = null;
for (int i = 0; i < len; i++) {
String command = commands.get(i);
if (i == 0) {
mSocket.setSoTimeout(SEARCH_TIMEOUT);
tag = writeCommand(mWriter, command);
} else {
writeContinuation(mWriter, command);
}
if (readResponse(mReader, tag, "SEARCH").equals(IMAP_OK)) {
// Done
String msgs = mImapResponse.get(0);
Parser p = new Parser(msgs, 8);
Integer[] serverList = p.gatherIntegers();
Arrays.sort(serverList, Collections.reverseOrder());
return new SearchResult(serverList, null);
} else if (mImapResult.startsWith("+")){
continue;
} else {
errorLog("Server doesn't understand complex SEARCH?");
break;
}
}
} catch (SocketTimeoutException e) {
exception = e;
errorLog("Search timed out");
} catch (IOException e) {
exception = e;
errorLog("Search IOException");
}
return new SearchResult(new Integer[0], exception);
}
public int searchMailbox(final Context context, long accountId, SearchParams searchParams,
final long destMailboxId) throws IOException {
final Account account = Account.restoreAccountWithId(context, accountId);
final Mailbox mailbox = Mailbox.restoreMailboxWithId(context, searchParams.mMailboxId);
final Mailbox destMailbox = Mailbox.restoreMailboxWithId(context, destMailboxId);
if (account == null || mailbox == null || destMailbox == null) {
Log.d(Logging.LOG_TAG, "Attempted search for " + searchParams
+ " but account or mailbox information was missing");
return 0;
}
HostAuth hostAuth = HostAuth.restoreHostAuthWithId(context, account.mHostAuthKeyRecv);
if (hostAuth == null) {
}
Connection conn = connectAndLogin(hostAuth, "search");
if (conn.status != EXIT_DONE) {
mExitStatus = conn.status;
return 0;
}
try {
setConnection(conn);
Integer[] sortedUids = null;
if (searchParams.mOffset == 0) {
SearchResult result = getSearchResults(searchParams);
if (result.exception == null) {
sortedUids = result.uids;
sSearchResults.put(accountId, sortedUids);
} else {
throw new IOException();
}
} else {
sortedUids = sSearchResults.get(accountId);
}
final int numSearchResults = sortedUids.length;
final int numToLoad =
Math.min(numSearchResults - searchParams.mOffset, searchParams.mLimit);
if (numToLoad <= 0) {
return 0;
}
final ArrayList<Integer> loadList = new ArrayList<Integer>();
for (int i = searchParams.mOffset; i < numToLoad + searchParams.mOffset; i++) {
loadList.add(sortedUids[i]);
}
try {
loadMessages(loadList, destMailboxId);
} catch (IOException e) {
// TODO: How do we handle this?
return 0;
}
return sortedUids.length;
} finally {
if (mSocket != null) {
try {
// Try to logout
readResponse(mReader, writeCommand(mWriter, "logout"));
mSocket.close();
} catch (IOException e) {
// We're leaving anyway
}
}
}
}
}

32
imap2/src/com/android/imap2/Parser.java
View File

@ -16,6 +16,8 @@
package com.android.imap2;
import java.util.ArrayList;
public class Parser {
String str;
int pos;
@ -154,7 +156,7 @@ public class Parser {
return str.substring(start, pos - 1);
}
public Integer parseInteger () {
public int parseInteger () {
skipWhite();
int start = pos;
while (pos < len) {
@ -165,14 +167,15 @@ public class Parser {
break;
}
if (pos > start) {
try {
Integer i = Integer.parseInt(str.substring(start, pos));
return i;
} catch (NumberFormatException e) {
return -1;
// We know these are positive integers
int sum = 0;
for (int i = start; i < pos; i++) {
sum = (sum * 10) + (str.charAt(i) - '0');
}
} else
return sum;
} else {
return -1;
}
}
public int[] gatherInts () {
@ -180,8 +183,7 @@ public class Parser {
int size = 128;
int offs = 0;
while (true) {
// TODO Slow; handle this inline rather than calling the method
Integer i = parseInteger();
int i = parseInteger();
if (i >= 0) {
if (offs == size) {
// Double the size of the array as necessary
@ -199,4 +201,16 @@ public class Parser {
System.arraycopy(list, 0, res, 0, offs);
return res;
}
public Integer[] gatherIntegers () {
ArrayList<Integer> list = new ArrayList<Integer>();
while (true) {
Integer i = parseInteger();
if (i >= 0) {
list.add(i);
}
else
break;
}
return list.toArray(new Integer[list.size()]);
}
}

43
imap2/src/com/android/imap2/SearchRequest.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.imap2;
import com.android.emailcommon.provider.EmailContent.Message;
import com.android.emailcommon.service.SearchParams;
import com.android.emailsync.Request;
/**
* SearchRequest is the wrapper for server search requests.
*/
public class SearchRequest extends Request {
public final SearchParams mParams;
public SearchRequest(SearchParams _params) {
super(Message.NO_MESSAGE);
mParams = _params;
}
// SearchRequests are unique by their mailboxId
public boolean equals(Object o) {
if (!(o instanceof SearchRequest)) return false;
return ((SearchRequest)o).mParams.mMailboxId == mParams.mMailboxId;
}
public int hashCode() {
return (int)mParams.mMailboxId;
}
}

4
src/com/android/email/provider/EmailProvider.java
View File

@ -3923,6 +3923,10 @@ outer:
private void notifyUIConversationMailbox(long id) {
notifyUI(UIPROVIDER_CONVERSATION_NOTIFIER, Long.toString(id));
Mailbox mailbox = Mailbox.restoreMailboxWithId(getContext(), id);
if (mailbox == null) {
Log.w(TAG, "No mailbox for notification: " + id);
return;
}
// Notify combined inbox...
if (mailbox.mType == Mailbox.TYPE_INBOX) {
notifyUI(UIPROVIDER_CONVERSATION_NOTIFIER,