diff --git a/emailcommon/src/com/android/emailcommon/utility/CountingOutputStream.java b/emailcommon/src/com/android/emailcommon/utility/CountingOutputStream.java index ec71634b9..034b8fd20 100644 --- a/emailcommon/src/com/android/emailcommon/utility/CountingOutputStream.java +++ b/emailcommon/src/com/android/emailcommon/utility/CountingOutputStream.java @@ -20,21 +20,30 @@ import java.io.IOException; import java.io.OutputStream; /** - * A simple OutputStream that does nothing but count how many bytes are written to it and + * A simple pass-thru OutputStream that also counts how many bytes are written to it and * makes that count available to callers. */ public class CountingOutputStream extends OutputStream { private long mCount; + private final OutputStream mOutputStream; - public CountingOutputStream() { + public CountingOutputStream(OutputStream outputStream) { + mOutputStream = outputStream; } public long getCount() { return mCount; } + @Override + public void write(byte[] buffer, int offset, int count) throws IOException { + mOutputStream.write(buffer, offset, count); + mCount += count; + } + @Override public void write(int oneByte) throws IOException { + mOutputStream.write(oneByte); mCount++; } } diff --git a/src/com/android/email/LegacyConversions.java b/src/com/android/email/LegacyConversions.java index 2c5d2ae17..574265e96 100644 --- a/src/com/android/email/LegacyConversions.java +++ b/src/com/android/email/LegacyConversions.java @@ -31,10 +31,12 @@ import com.android.emailcommon.internet.MimeMultipart; import com.android.emailcommon.internet.MimeUtility; import com.android.emailcommon.internet.TextBody; import com.android.emailcommon.mail.Address; +import com.android.emailcommon.mail.Base64Body; import com.android.emailcommon.mail.Flag; import com.android.emailcommon.mail.Message; import com.android.emailcommon.mail.Message.RecipientType; import com.android.emailcommon.mail.MessagingException; +import com.android.emailcommon.mail.Multipart; import com.android.emailcommon.mail.Part; import com.android.emailcommon.provider.EmailContent; import com.android.emailcommon.provider.EmailContent.Attachment; @@ -46,7 +48,9 @@ import com.android.mail.utils.LogUtils; import org.apache.commons.io.IOUtils; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -421,15 +425,39 @@ public class LegacyConversions { } // Attachments - // TODO: Make sure we deal with these as structures and don't accidentally upload files -// Uri uri = ContentUris.withAppendedId(Attachment.MESSAGE_ID_URI, localMessage.mId); -// Cursor attachments = context.getContentResolver().query(uri, Attachment.CONTENT_PROJECTION, -// null, null, null); -// try { -// -// } finally { -// attachments.close(); -// } + Uri uri = ContentUris.withAppendedId(Attachment.MESSAGE_ID_URI, localMessage.mId); + Cursor attachments = context.getContentResolver().query(uri, Attachment.CONTENT_PROJECTION, + null, null, null); + + try { + while (attachments != null && attachments.moveToNext()) { + final Attachment att = new Attachment(); + att.restore(attachments); + try { + final InputStream content; + if (att.mContentBytes != null) { + // This is generally only the case for synthetic attachments, such as those + // generated by unit tests or calendar invites + content = new ByteArrayInputStream(att.mContentBytes); + } else { + final Uri contentUri = Uri.parse(att.getCachedFileUri()); + content = context.getContentResolver().openInputStream(contentUri); + } + final String mimeType = att.mMimeType; + final Long contentSize = att.mSize; + final String contentId = att.mContentId; + final String filename = att.mFileName; + addAttachmentPart(mp, mimeType, contentSize, filename, contentId, content); + } catch (final FileNotFoundException e) { + LogUtils.e(LogUtils.TAG, "File Not Found error on %s while upsyncing message", + att.getCachedFileUri()); + } + } + } finally { + if (attachments != null) { + attachments.close(); + } + } return message; } @@ -455,6 +483,31 @@ public class LegacyConversions { mp.addBodyPart(bp); } + /** + * Helper method to add an attachment part + * + * @param mp Multipart message to append attachment part to + * @param contentType Mime type + * @param contentSize Attachment metadata: unencoded file size + * @param filename Attachment metadata: file name + * @param contentId as referenced from cid: uris in the message body (if applicable) + * @param content unencoded bytes + * @throws MessagingException + */ + private static void addAttachmentPart(final Multipart mp, final String contentType, + final Long contentSize, final String filename, final String contentId, + final InputStream content) throws MessagingException { + final Base64Body body = new Base64Body(content); + final MimeBodyPart bp = new MimeBodyPart(body, contentType); + bp.setHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING, "base64"); + bp.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION, "attachment;\n" + + "filename=\"" + filename + "\";" + + "size=" + contentSize); + if (contentId != null) { + bp.setHeader(MimeHeader.HEADER_CONTENT_ID, contentId); + } + mp.addBodyPart(bp); + } /** * Infer mailbox type from mailbox name. Used by MessagingController (for live folder sync). diff --git a/src/com/android/email/mail/store/ImapFolder.java b/src/com/android/email/mail/store/ImapFolder.java index 30857360c..363b49b0c 100644 --- a/src/com/android/email/mail/store/ImapFolder.java +++ b/src/com/android/email/mail/store/ImapFolder.java @@ -52,6 +52,11 @@ import com.android.emailcommon.utility.Utility; import com.android.mail.utils.LogUtils; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -1025,92 +1030,124 @@ class ImapFolder extends Folder { * Appends the given messages to the selected folder. This implementation also determines * the new UID of the given message on the IMAP server and sets the Message's UID to the * new server UID. + * @param message Message + * @param noTimeout Set to true on manual syncs, disables the timeout after sending the message + * content to the server */ @Override - public void appendMessages(Message[] messages) throws MessagingException { + public void appendMessage(final Context context, final Message message, final boolean noTimeout) + throws MessagingException { checkOpen(); try { - for (Message message : messages) { - // Create output count - CountingOutputStream out = new CountingOutputStream(); - EOLConvertingOutputStream eolOut = new EOLConvertingOutputStream(out); - message.writeTo(eolOut); - eolOut.flush(); - // Create flag list (most often this will be "\SEEN") - String flagList = ""; - Flag[] flags = message.getFlags(); - if (flags.length > 0) { - StringBuilder sb = new StringBuilder(); - for (int i = 0, count = flags.length; i < count; i++) { - Flag flag = flags[i]; - if (flag == Flag.SEEN) { - sb.append(" " + ImapConstants.FLAG_SEEN); - } else if (flag == Flag.FLAGGED) { - sb.append(" " + ImapConstants.FLAG_FLAGGED); - } - } - if (sb.length() > 0) { - flagList = sb.substring(1); + // Create temp file + /** + * We need to know the encoded message size before we upload it, and encoding + * attachments as Base64, possibly reading from a slow provider, is a non-trivial + * operation. So we write the contents to a temp file while measuring the size, + * and then use that temp file and size to do the actual upsync. + * For context, most classic email clients would store the message in RFC822 format + * internally, and so would not need to do this on-the-fly. + */ + final File tempDir = context.getExternalCacheDir(); + final File tempFile = File.createTempFile("IMAPupsync", ".eml", tempDir); + // Delete here so we don't leave the file lingering. We've got a handle to it so we + // can still use it. + final boolean deleteSuccessful = tempFile.delete(); + if (!deleteSuccessful) { + LogUtils.w(LogUtils.TAG, "Could not delete temp file %s", + tempFile.getAbsolutePath()); + } + final OutputStream tempOut = new FileOutputStream(tempFile); + // Create output count while writing temp file + final CountingOutputStream out = new CountingOutputStream(tempOut); + final EOLConvertingOutputStream eolOut = new EOLConvertingOutputStream(out); + message.writeTo(eolOut); + eolOut.flush(); + // Create flag list (most often this will be "\SEEN") + String flagList = ""; + Flag[] flags = message.getFlags(); + if (flags.length > 0) { + StringBuilder sb = new StringBuilder(); + for (final Flag flag : flags) { + if (flag == Flag.SEEN) { + sb.append(" " + ImapConstants.FLAG_SEEN); + } else if (flag == Flag.FLAGGED) { + sb.append(" " + ImapConstants.FLAG_FLAGGED); } } + if (sb.length() > 0) { + flagList = sb.substring(1); + } + } - mConnection.sendCommand( - String.format(Locale.US, ImapConstants.APPEND + " \"%s\" (%s) {%d}", - ImapStore.encodeFolderName(mName, mStore.mPathPrefix), - flagList, - out.getCount()), false); - ImapResponse response; - do { + mConnection.sendCommand( + String.format(Locale.US, ImapConstants.APPEND + " \"%s\" (%s) {%d}", + ImapStore.encodeFolderName(mName, mStore.mPathPrefix), + flagList, + out.getCount()), false); + ImapResponse response; + do { + final int socketTimeout = mConnection.mTransport.getSoTimeout(); + try { + // Need to set the timeout to unlimited since we might be upsyncing a pretty + // big attachment so who knows how long it'll take. It would sure be nice + // if this only timed out after the send buffer drained but welp. + if (noTimeout) { + // For now, only unset the timeout if we're doing a manual sync + mConnection.mTransport.setSoTimeout(0); + } response = mConnection.readResponse(); if (response.isContinuationRequest()) { - eolOut = new EOLConvertingOutputStream( - mConnection.mTransport.getOutputStream()); - message.writeTo(eolOut); - eolOut.write('\r'); - eolOut.write('\n'); - eolOut.flush(); + final OutputStream transportOutputStream = + mConnection.mTransport.getOutputStream(); + IOUtils.copyLarge(new FileInputStream(tempFile), transportOutputStream); + transportOutputStream.write('\r'); + transportOutputStream.write('\n'); + transportOutputStream.flush(); } else if (!response.isTagged()) { handleUntaggedResponse(response); } - } while (!response.isTagged()); + } finally { + mConnection.mTransport.setSoTimeout(socketTimeout); + } + } while (!response.isTagged()); - // TODO Why not check the response? + // TODO Why not check the response? /* * Try to recover the UID of the message from an APPENDUID response. * e.g. 11 OK [APPENDUID 2 238268] APPEND completed */ - final ImapList appendList = response.getListOrEmpty(1); - if ((appendList.size() >= 3) && appendList.is(0, ImapConstants.APPENDUID)) { - String serverUid = appendList.getStringOrEmpty(2).getString(); - if (!TextUtils.isEmpty(serverUid)) { - message.setUid(serverUid); - continue; - } + final ImapList appendList = response.getListOrEmpty(1); + if ((appendList.size() >= 3) && appendList.is(0, ImapConstants.APPENDUID)) { + String serverUid = appendList.getStringOrEmpty(2).getString(); + if (!TextUtils.isEmpty(serverUid)) { + message.setUid(serverUid); + return; } + } - /* - * Try to find the UID of the message we just appended using the - * Message-ID header. If there are more than one response, take the - * last one, as it's most likely the newest (the one we just uploaded). - */ - final String messageId = message.getMessageId(); - if (messageId == null || messageId.length() == 0) { - continue; - } - // Most servers don't care about parenthesis in the search query [and, some - // fail to work if they are used] - String[] uids = searchForUids( - String.format(Locale.US, "HEADER MESSAGE-ID %s", messageId)); - if (uids.length > 0) { - message.setUid(uids[0]); - } - // However, there's at least one server [AOL] that fails to work unless there - // are parenthesis, so, try this as a last resort - uids = searchForUids(String.format(Locale.US, "(HEADER MESSAGE-ID %s)", messageId)); - if (uids.length > 0) { - message.setUid(uids[0]); - } + /* + * Try to find the UID of the message we just appended using the + * Message-ID header. If there are more than one response, take the + * last one, as it's most likely the newest (the one we just uploaded). + */ + final String messageId = message.getMessageId(); + if (messageId == null || messageId.length() == 0) { + return; + } + // Most servers don't care about parenthesis in the search query [and, some + // fail to work if they are used] + String[] uids = searchForUids( + String.format(Locale.US, "HEADER MESSAGE-ID %s", messageId)); + if (uids.length > 0) { + message.setUid(uids[0]); + } + // However, there's at least one server [AOL] that fails to work unless there + // are parenthesis, so, try this as a last resort + uids = searchForUids(String.format(Locale.US, "(HEADER MESSAGE-ID %s)", messageId)); + if (uids.length > 0) { + message.setUid(uids[0]); } } catch (IOException ioe) { throw ioExceptionHandler(mConnection, ioe); diff --git a/src/com/android/email/mail/store/Pop3Store.java b/src/com/android/email/mail/store/Pop3Store.java index cb959fdd0..d3f80653a 100644 --- a/src/com/android/email/mail/store/Pop3Store.java +++ b/src/com/android/email/mail/store/Pop3Store.java @@ -553,7 +553,7 @@ public class Pop3Store extends Store { * * @param message * @param lines - * @param optional callback that reports progress of the fetch + * @param callback optional callback that reports progress of the fetch */ public void fetchBody(Pop3Message message, int lines, EOLConvertingInputStream.Callback callback) throws IOException, MessagingException { @@ -626,7 +626,7 @@ public class Pop3Store extends Store { } @Override - public void appendMessages(Message[] messages) { + public void appendMessage(Context context, Message message, boolean noTimeout) { } @Override diff --git a/src/com/android/email/mail/transport/MailTransport.java b/src/com/android/email/mail/transport/MailTransport.java index d47c831d6..f34127de6 100644 --- a/src/com/android/email/mail/transport/MailTransport.java +++ b/src/com/android/email/mail/transport/MailTransport.java @@ -209,6 +209,15 @@ public class MailTransport { } } + /** + * Get the socket timeout. + * @return the read timeout value in milliseconds + * @throws SocketException + */ + public int getSoTimeout() throws SocketException { + return mSocket.getSoTimeout(); + } + /** * Set the socket timeout. * @param timeoutMilliseconds the read timeout value if greater than {@code 0}, or diff --git a/src/com/android/email/service/ImapService.java b/src/com/android/email/service/ImapService.java index 255dfefe3..526a758ab 100644 --- a/src/com/android/email/service/ImapService.java +++ b/src/com/android/email/service/ImapService.java @@ -164,7 +164,7 @@ public class ImapService extends Service { Store remoteStore = null; try { remoteStore = Store.getInstance(account, context); - processPendingActionsSynchronous(context, account, remoteStore); + processPendingActionsSynchronous(context, account, remoteStore, uiRefresh); synchronizeMailboxGeneric(context, account, remoteStore, folder, loadMore, uiRefresh); // Clear authentication notification for this account nc.cancelLoginFailedNotification(account.mId); @@ -724,7 +724,7 @@ public class ImapService extends Service { * @throws MessagingException */ private static void processPendingActionsSynchronous(Context context, Account account, - Store remoteStore) + Store remoteStore, boolean manualSync) throws MessagingException { TrafficStats.setThreadStatsTag(TrafficFlags.getSyncFlags(context, account)); String[] accountIdArgs = new String[] { Long.toString(account.mId) }; @@ -733,7 +733,7 @@ public class ImapService extends Service { processPendingDeletesSynchronous(context, account, remoteStore, accountIdArgs); // Handle uploads (currently, only to sent messages) - processPendingUploadsSynchronous(context, account, remoteStore, accountIdArgs); + processPendingUploadsSynchronous(context, account, remoteStore, accountIdArgs, manualSync); // Now handle updates / upsyncs processPendingUpdatesSynchronous(context, account, remoteStore, accountIdArgs); @@ -843,7 +843,7 @@ public class ImapService extends Service { * uploaded directly to the Sent folder. */ private static void processPendingUploadsSynchronous(Context context, Account account, - Store remoteStore, String[] accountIdArgs) { + Store remoteStore, String[] accountIdArgs, boolean manualSync) { ContentResolver resolver = context.getContentResolver(); // Find the Sent folder (since that's all we're uploading for now // TODO: Upsync for all folders? (In case a user moves mail from Sent before it is @@ -884,7 +884,7 @@ public class ImapService extends Service { // upsync the message long id = upsyncs1.getLong(EmailContent.Message.ID_PROJECTION_COLUMN); lastMessageId = id; - processUploadMessage(context, remoteStore, mailbox, id); + processUploadMessage(context, remoteStore, mailbox, id, manualSync); } } finally { if (upsyncs1 != null) { @@ -1005,7 +1005,7 @@ public class ImapService extends Service { * @param mailbox the actual mailbox */ private static void processUploadMessage(Context context, Store remoteStore, Mailbox mailbox, - long messageId) + long messageId, boolean manualSync) throws MessagingException { EmailContent.Message newMessage = EmailContent.Message.restoreMessageWithId(context, messageId); @@ -1026,8 +1026,9 @@ public class ImapService extends Service { deleteUpdate = false; LogUtils.d(Logging.LOG_TAG, "Upsync skipped; mailbox changed, id=" + messageId); } else { - LogUtils.d(Logging.LOG_TAG, "Upsyc triggered for message id=" + messageId); - deleteUpdate = processPendingAppend(context, remoteStore, mailbox, newMessage); + LogUtils.d(Logging.LOG_TAG, "Upsync triggered for message id=" + messageId); + deleteUpdate = + processPendingAppend(context, remoteStore, mailbox, newMessage, manualSync); } if (deleteUpdate) { // Finally, delete the update (if any) @@ -1286,10 +1287,11 @@ public class ImapService extends Service { * @param remoteStore the remote store we're working in * @param mailbox The mailbox we're appending to * @param message The message we're appending + * @param manualSync True if this is a manual sync (changes upsync behavior) * @return true if successfully uploaded */ private static boolean processPendingAppend(Context context, Store remoteStore, Mailbox mailbox, - EmailContent.Message message) + EmailContent.Message message, boolean manualSync) throws MessagingException { boolean updateInternalDate = false; boolean updateMessage = false; @@ -1325,7 +1327,7 @@ public class ImapService extends Service { //FetchProfile fp = new FetchProfile(); //fp.add(FetchProfile.Item.BODY); // Note that this operation will assign the Uid to localMessage - remoteFolder.appendMessages(new Message[] { localMessage }); + remoteFolder.appendMessage(context, localMessage, manualSync /* no timeout */); // 3b. And record the UID from the server message.mServerId = localMessage.getUid(); @@ -1360,7 +1362,7 @@ public class ImapService extends Service { fp.clear(); fp = new FetchProfile(); fp.add(FetchProfile.Item.BODY); - remoteFolder.appendMessages(new Message[] { localMessage }); + remoteFolder.appendMessage(context, localMessage, manualSync /* no timeout */); // 4d. Record the UID and new internalDate from the server message.mServerId = localMessage.getUid(); diff --git a/src/com/android/email/service/PopImapSyncAdapterService.java b/src/com/android/email/service/PopImapSyncAdapterService.java index 4b62860e0..e07a6ed31 100644 --- a/src/com/android/email/service/PopImapSyncAdapterService.java +++ b/src/com/android/email/service/PopImapSyncAdapterService.java @@ -243,7 +243,8 @@ public class PopImapSyncAdapterService extends Service { int deltaMessageCount = extras.getInt(Mailbox.SYNC_EXTRA_DELTA_MESSAGE_COUNT, 0); for (long mailboxId : mailboxIds) { - sync(context, mailboxId, extras, syncResult, uiRefresh, deltaMessageCount); + sync(context, mailboxId, extras, syncResult, uiRefresh, + deltaMessageCount); } } } diff --git a/tests/src/com/android/email/mail/store/ImapStoreUnitTests.java b/tests/src/com/android/email/mail/store/ImapStoreUnitTests.java index 2c1c6dc99..5e9efe617 100644 --- a/tests/src/com/android/email/mail/store/ImapStoreUnitTests.java +++ b/tests/src/com/android/email/mail/store/ImapStoreUnitTests.java @@ -1185,7 +1185,7 @@ public class ImapStoreUnitTests extends InstrumentationTestCase { ImapMessage message = prepareForAppendTest(mock, "oK [aPPENDUID 1234567 13] (Success)"); - mFolder.appendMessages(new Message[] {message}); + mFolder.appendMessage(getInstrumentation().getTargetContext(), message, false); assertEquals("13", message.getUid()); assertEquals(7, mFolder.getMessageCount()); @@ -1216,7 +1216,7 @@ public class ImapStoreUnitTests extends InstrumentationTestCase { getNextTag(true) + " oK success" }); - mFolder.appendMessages(new Message[] {message}); + mFolder.appendMessage(getInstrumentation().getTargetContext(), message, false); assertEquals("321", message.getUid()); } @@ -1250,7 +1250,7 @@ public class ImapStoreUnitTests extends InstrumentationTestCase { getNextTag(true) + " oK Search completed." }); - mFolder.appendMessages(new Message[] {message}); + mFolder.appendMessage(getInstrumentation().getTargetContext(), message, false); // Shouldn't have changed assertEquals("initial uid", message.getUid()); diff --git a/tests/src/com/android/email/mail/store/Pop3StoreUnitTests.java b/tests/src/com/android/email/mail/store/Pop3StoreUnitTests.java index 58283d442..0af788756 100644 --- a/tests/src/com/android/email/mail/store/Pop3StoreUnitTests.java +++ b/tests/src/com/android/email/mail/store/Pop3StoreUnitTests.java @@ -298,9 +298,6 @@ public class Pop3StoreUnitTests extends AndroidTestCase { assertEquals(1, flags.length); assertEquals(Flag.DELETED, flags[0]); - // appendMessages(Message[] messages) does nothing - mFolder.appendMessages(null); - // delete(boolean recurse) does nothing // TODO - it should! mFolder.delete(false); diff --git a/tests/src/com/android/emailcommon/mail/MockFolder.java b/tests/src/com/android/emailcommon/mail/MockFolder.java index a35fb3db2..9c5ae7a4c 100644 --- a/tests/src/com/android/emailcommon/mail/MockFolder.java +++ b/tests/src/com/android/emailcommon/mail/MockFolder.java @@ -16,13 +16,15 @@ package com.android.emailcommon.mail; +import android.content.Context; + import com.android.emailcommon.service.SearchParams; public class MockFolder extends Folder { @Override - public void appendMessages(Message[] messages) { + public void appendMessage(Context context, Message message, boolean noTimeout) { } @Override