From 20225d57609d6a5e482c088fdad60c29212d31a0 Mon Sep 17 00:00:00 2001 From: Makoto Onuki Date: Fri, 12 Mar 2010 13:30:26 -0800 Subject: [PATCH] Explicitly send ICS files in UTF-8. - In memory attachments are now stored as byte[], not String. We can store any type of contents now. - Added blob content_bytes to the Attachment table. The content field is now deprecated and not used. - Explicitly convert ICS files to UTF-8. - Added Utility.to/fromUtf8(). Bug 2509287 Change-Id: I3785a365a9a34039ec12ba82bd857dcdbc4de92d --- src/com/android/email/Utility.java | 17 ++++++ .../email/mail/transport/Rfc822Output.java | 4 +- .../android/email/provider/EmailContent.java | 29 ++++++++-- .../android/email/provider/EmailProvider.java | 17 +++++- .../exchange/utility/CalendarUtilities.java | 8 ++- tests/src/com/android/email/TestUtils.java | 57 +++++++++++++++++++ .../com/android/email/UtilityUnitTests.java | 12 ++++ .../email/provider/ProviderTestUtils.java | 5 ++ .../utility/CalendarUtilitiesTests.java | 17 +++--- 9 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 tests/src/com/android/email/TestUtils.java diff --git a/src/com/android/email/Utility.java b/src/com/android/email/Utility.java index 9f0cc84bd..a3a745914 100644 --- a/src/com/android/email/Utility.java +++ b/src/com/android/email/Utility.java @@ -32,6 +32,7 @@ import android.content.res.TypedArray; import android.database.Cursor; import android.graphics.drawable.Drawable; import android.text.Editable; +import android.util.Log; import android.util.base64.Base64; import android.widget.TextView; @@ -39,11 +40,16 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; import java.util.Date; import java.util.GregorianCalendar; import java.util.TimeZone; public class Utility { + public static final Charset UTF_8 = Charset.forName("UTF-8"); + public final static String readInputStream(InputStream in, String encoding) throws IOException { InputStreamReader reader = new InputStreamReader(in, encoding); StringBuffer sb = new StringBuffer(); @@ -458,4 +464,15 @@ public class Utility { cal.setTimeZone(TimeZone.getTimeZone("GMT")); return cal.getTimeInMillis(); } + + /** Converts a String to UTF-8 */ + public static byte[] toUtf8(String s) { + if (s == null) { + return null; + } + final ByteBuffer buffer = UTF_8.encode(CharBuffer.wrap(s)); + final byte[] bytes = new byte[buffer.limit()]; + buffer.get(bytes); + return bytes; + } } diff --git a/src/com/android/email/mail/transport/Rfc822Output.java b/src/com/android/email/mail/transport/Rfc822Output.java index 8c0dcf86c..cdd821add 100644 --- a/src/com/android/email/mail/transport/Rfc822Output.java +++ b/src/com/android/email/mail/transport/Rfc822Output.java @@ -226,8 +226,8 @@ public class Rfc822Output { InputStream inStream = null; try { // Use content, if provided; otherwise, use the contentUri - if (attachment.mContent != null) { - inStream = new ByteArrayInputStream(attachment.mContent.getBytes()); + if (attachment.mContentBytes != null) { + inStream = new ByteArrayInputStream(attachment.mContentBytes); } else { // try to open the file Uri fileUri = Uri.parse(attachment.mContentUri); diff --git a/src/com/android/email/provider/EmailContent.java b/src/com/android/email/provider/EmailContent.java index 07bfe89fb..1e5f3ba49 100644 --- a/src/com/android/email/provider/EmailContent.java +++ b/src/com/android/email/provider/EmailContent.java @@ -1633,10 +1633,12 @@ public abstract class EmailContent { public static final String LOCATION = "location"; // The transfer encoding of the attachment public static final String ENCODING = "encoding"; - // Content that is actually contained in the Attachment row + // Not currently used public static final String CONTENT = "content"; // Flags public static final String FLAGS = "flags"; + // Content that is actually contained in the Attachment row + public static final String CONTENT_BYTES = "content_bytes"; } public static final class Attachment extends EmailContent implements AttachmentColumns { @@ -1654,8 +1656,9 @@ public abstract class EmailContent { public long mMessageKey; public String mLocation; public String mEncoding; - public String mContent; + public String mContent; // Not currently used public int mFlags; + public byte[] mContentBytes; public static final int CONTENT_ID_COLUMN = 0; public static final int CONTENT_FILENAME_COLUMN = 1; @@ -1666,13 +1669,14 @@ public abstract class EmailContent { public static final int CONTENT_MESSAGE_ID_COLUMN = 6; public static final int CONTENT_LOCATION_COLUMN = 7; public static final int CONTENT_ENCODING_COLUMN = 8; - public static final int CONTENT_CONTENT_COLUMN = 9; + public static final int CONTENT_CONTENT_COLUMN = 9; // Not currently used public static final int CONTENT_FLAGS_COLUMN = 10; + public static final int CONTENT_CONTENT_BYTES_COLUMN = 11; public static final String[] CONTENT_PROJECTION = new String[] { RECORD_ID, AttachmentColumns.FILENAME, AttachmentColumns.MIME_TYPE, AttachmentColumns.SIZE, AttachmentColumns.CONTENT_ID, AttachmentColumns.CONTENT_URI, AttachmentColumns.MESSAGE_KEY, AttachmentColumns.LOCATION, AttachmentColumns.ENCODING, - AttachmentColumns.CONTENT, AttachmentColumns.FLAGS + AttachmentColumns.CONTENT, AttachmentColumns.FLAGS, AttachmentColumns.CONTENT_BYTES }; // Bits used in mFlags @@ -1777,6 +1781,7 @@ public abstract class EmailContent { mEncoding = cursor.getString(CONTENT_ENCODING_COLUMN); mContent = cursor.getString(CONTENT_CONTENT_COLUMN); mFlags = cursor.getInt(CONTENT_FLAGS_COLUMN); + mContentBytes = cursor.getBlob(CONTENT_CONTENT_BYTES_COLUMN); return this; } @@ -1793,6 +1798,7 @@ public abstract class EmailContent { values.put(AttachmentColumns.ENCODING, mEncoding); values.put(AttachmentColumns.CONTENT, mContent); values.put(AttachmentColumns.FLAGS, mFlags); + values.put(AttachmentColumns.CONTENT_BYTES, mContentBytes); return values; } @@ -1813,6 +1819,12 @@ public abstract class EmailContent { dest.writeString(mEncoding); dest.writeString(mContent); dest.writeInt(mFlags); + if (mContentBytes == null) { + dest.writeInt(-1); + } else { + dest.writeInt(mContentBytes.length); + dest.writeByteArray(mContentBytes); + } } public Attachment(Parcel in) { @@ -1828,6 +1840,13 @@ public abstract class EmailContent { mEncoding = in.readString(); mContent = in.readString(); mFlags = in.readInt(); + final int contentBytesLen = in.readInt(); + if (contentBytesLen == -1) { + mContentBytes = null; + } else { + mContentBytes = new byte[contentBytesLen]; + in.readByteArray(mContentBytes); + } } public static final Parcelable.Creator CREATOR @@ -1845,7 +1864,7 @@ public abstract class EmailContent { public String toString() { return "[" + mFileName + ", " + mMimeType + ", " + mSize + ", " + mContentId + ", " + mContentUri + ", " + mMessageKey + ", " + mLocation + ", " + mEncoding + ", " - + mContent + ", " + mFlags + "]"; + + mFlags + ", " + mContentBytes + "]"; } } diff --git a/src/com/android/email/provider/EmailProvider.java b/src/com/android/email/provider/EmailProvider.java index b360b1852..783e85240 100644 --- a/src/com/android/email/provider/EmailProvider.java +++ b/src/com/android/email/provider/EmailProvider.java @@ -80,7 +80,8 @@ public class EmailProvider extends ContentProvider { // Version 9: Add security sync key and signature to accounts table // Version 10: Add meeting info to message table // Version 11: Add content and flags to attachment table - public static final int DATABASE_VERSION = 11; + // Version 12: Add content_bytes to attachment table. content is deprecated. + public static final int DATABASE_VERSION = 12; // Any changes to the database format *must* include update-in-place code. // Original version: 2 @@ -506,7 +507,8 @@ public class EmailProvider extends ContentProvider { + AttachmentColumns.LOCATION + " text, " + AttachmentColumns.ENCODING + " text, " + AttachmentColumns.CONTENT + " text, " - + AttachmentColumns.FLAGS + " integer" + + AttachmentColumns.FLAGS + " integer, " + + AttachmentColumns.CONTENT_BYTES + " blob" + ");"; db.execSQL("create table " + Attachment.TABLE_NAME + s); db.execSQL(createIndex(Attachment.TABLE_NAME, AttachmentColumns.MESSAGE_KEY)); @@ -769,6 +771,17 @@ public class EmailProvider extends ContentProvider { } oldVersion = 11; } + if (oldVersion == 11) { + // Attachment: add content_bytes + try { + db.execSQL("alter table " + Attachment.TABLE_NAME + + " add column " + AttachmentColumns.CONTENT_BYTES + " blob" + ";"); + } catch (SQLException e) { + // Shouldn't be needed unless we're debugging and interrupt the process + Log.w(TAG, "Exception upgrading EmailProvider.db from 11 to 12 " + e); + } + oldVersion = 12; + } } @Override diff --git a/src/com/android/exchange/utility/CalendarUtilities.java b/src/com/android/exchange/utility/CalendarUtilities.java index 8b27eae40..db1a2b9f5 100644 --- a/src/com/android/exchange/utility/CalendarUtilities.java +++ b/src/com/android/exchange/utility/CalendarUtilities.java @@ -18,6 +18,7 @@ package com.android.exchange.utility; import com.android.email.Email; import com.android.email.R; +import com.android.email.Utility; import com.android.email.mail.Address; import com.android.email.provider.EmailContent; import com.android.email.provider.EmailContent.Account; @@ -1458,10 +1459,13 @@ public class CalendarUtilities { // Create the ics attachment using the "content" field Attachment att = new Attachment(); - att.mContent = ics.toString(); + + // TODO UTF-8 conversion should be done in SimpleIcsWriter, as it should count line + // length for folding in bytes in UTF-8. + att.mContentBytes = Utility.toUtf8(ics.toString()); att.mMimeType = "text/calendar; method=" + method; att.mFileName = "invite.ics"; - att.mSize = att.mContent.length(); + att.mSize = att.mContentBytes.length; // We don't send content-disposition with this attachment att.mFlags = Attachment.FLAG_SUPPRESS_DISPOSITION; diff --git a/tests/src/com/android/email/TestUtils.java b/tests/src/com/android/email/TestUtils.java new file mode 100644 index 000000000..5972d797c --- /dev/null +++ b/tests/src/com/android/email/TestUtils.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.email; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; + +import junit.framework.TestCase; + +/** + * Utility methods used only by tests. + */ +public class TestUtils extends TestCase /* It tests itself */ { + /** Shortcut to create byte array */ + public static byte[] b(int... array) { + if (array == null) { + return null; + } + byte[] ret = new byte[array.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (byte) array[i]; + } + return ret; + } + + /** Converts a String from UTF-8 */ + public static String fromUtf8(byte[] b) { + if (b == null) { + return null; + } + final CharBuffer cb = Utility.UTF_8.decode(ByteBuffer.wrap(b)); + return new String(cb.array(), 0, cb.length()); + } + + public void testUtf8() { + assertNull(fromUtf8(null)); + assertEquals("", fromUtf8(new byte[] {})); + assertEquals("a", fromUtf8(b('a'))); + assertEquals("ABC", fromUtf8(b('A', 'B', 'C'))); + assertEquals("\u65E5\u672C\u8A9E", + fromUtf8(b(0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC, 0xE8, 0xAA, 0x9E))); + } +} diff --git a/tests/src/com/android/email/UtilityUnitTests.java b/tests/src/com/android/email/UtilityUnitTests.java index 7b75c911d..16608943e 100644 --- a/tests/src/com/android/email/UtilityUnitTests.java +++ b/tests/src/com/android/email/UtilityUnitTests.java @@ -21,6 +21,7 @@ import com.android.email.provider.EmailContent.Mailbox; import android.content.Context; import android.graphics.drawable.Drawable; import android.test.AndroidTestCase; +import android.test.MoreAsserts; import android.test.suitebuilder.annotation.SmallTest; import java.util.HashSet; @@ -112,4 +113,15 @@ public class UtilityUnitTests extends AndroidTestCase { set.add(junk); assertEquals(8, set.size()); } + + + /** Test for {@link Utility#toUtf8} and {@link Utility#fromUtf8} */ + public void testUtf8() { + assertNull(Utility.toUtf8(null)); + MoreAsserts.assertEquals(new byte[] {}, Utility.toUtf8("")); + MoreAsserts.assertEquals(TestUtils.b('a'), Utility.toUtf8("a")); + MoreAsserts.assertEquals(TestUtils.b('A', 'B', 'C'), Utility.toUtf8("ABC")); + MoreAsserts.assertEquals(TestUtils.b(0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC, 0xE8, 0xAA, 0x9E), + Utility.toUtf8("\u65E5\u672C\u8A9E")); + } } diff --git a/tests/src/com/android/email/provider/ProviderTestUtils.java b/tests/src/com/android/email/provider/ProviderTestUtils.java index 22696fc6c..5caca393b 100644 --- a/tests/src/com/android/email/provider/ProviderTestUtils.java +++ b/tests/src/com/android/email/provider/ProviderTestUtils.java @@ -16,6 +16,7 @@ package com.android.email.provider; +import com.android.email.Utility; import com.android.email.provider.EmailContent.Account; import com.android.email.provider.EmailContent.Attachment; import com.android.email.provider.EmailContent.HostAuth; @@ -23,6 +24,7 @@ import com.android.email.provider.EmailContent.Mailbox; import com.android.email.provider.EmailContent.Message; import android.content.Context; +import android.test.MoreAsserts; import junit.framework.Assert; @@ -200,6 +202,7 @@ public class ProviderTestUtils extends Assert { att.mEncoding = "encoding " + fileName; att.mContent = "content " + fileName; att.mFlags = 0; + att.mContentBytes = Utility.toUtf8("content " + fileName); if (saveIt) { att.save(context); } @@ -364,5 +367,7 @@ public class ProviderTestUtils extends Assert { assertEquals(caller + " mEncoding", expect.mEncoding, actual.mEncoding); assertEquals(caller + " mContent", expect.mContent, actual.mContent); assertEquals(caller + " mFlags", expect.mFlags, actual.mFlags); + MoreAsserts.assertEquals(caller + " mContentBytes", + expect.mContentBytes, actual.mContentBytes); } } diff --git a/tests/src/com/android/exchange/utility/CalendarUtilitiesTests.java b/tests/src/com/android/exchange/utility/CalendarUtilitiesTests.java index a558a3b66..b766800ff 100644 --- a/tests/src/com/android/exchange/utility/CalendarUtilitiesTests.java +++ b/tests/src/com/android/exchange/utility/CalendarUtilitiesTests.java @@ -18,6 +18,7 @@ package com.android.exchange.utility; import com.android.email.R; import com.android.email.Utility; +import com.android.email.TestUtils; import com.android.email.mail.Address; import com.android.email.provider.EmailContent.Account; import com.android.email.provider.EmailContent.Attachment; @@ -231,7 +232,8 @@ public class CalendarUtilitiesTests extends AndroidTestCase { assertEquals(Attachment.FLAG_SUPPRESS_DISPOSITION, att.mFlags & Attachment.FLAG_SUPPRESS_DISPOSITION); assertEquals("text/calendar; method=REPLY", att.mMimeType); - assertNotNull(att.mContent); + assertNotNull(att.mContentBytes); + assertEquals(att.mSize, att.mContentBytes.length); //TODO Check the contents of the attachment using an iCalendar parser } @@ -271,10 +273,11 @@ public class CalendarUtilitiesTests extends AndroidTestCase { assertEquals(Attachment.FLAG_SUPPRESS_DISPOSITION, att.mFlags & Attachment.FLAG_SUPPRESS_DISPOSITION); assertEquals("text/calendar; method=REQUEST", att.mMimeType); - assertNotNull(att.mContent); + assertNotNull(att.mContentBytes); + assertEquals(att.mSize, att.mContentBytes.length); // We'll check the contents of the ics file here - BlockHash vcalendar = parseIcsContent(att.mContent); + BlockHash vcalendar = parseIcsContent(att.mContentBytes); assertNotNull(vcalendar); // We should have a VCALENDAR with a REQUEST method @@ -338,10 +341,10 @@ public class CalendarUtilitiesTests extends AndroidTestCase { assertEquals(Attachment.FLAG_SUPPRESS_DISPOSITION, att.mFlags & Attachment.FLAG_SUPPRESS_DISPOSITION); assertEquals("text/calendar; method=REQUEST", att.mMimeType); - assertNotNull(att.mContent); + assertNotNull(att.mContentBytes); // We'll check the contents of the ics file here - BlockHash vcalendar = parseIcsContent(att.mContent); + BlockHash vcalendar = parseIcsContent(att.mContentBytes); assertNotNull(vcalendar); // We should have a VCALENDAR with a REQUEST method @@ -537,8 +540,8 @@ public class CalendarUtilitiesTests extends AndroidTestCase { } } - private BlockHash parseIcsContent(String s) throws IOException { - BufferedReader reader = new BufferedReader(new StringReader(s)); + private BlockHash parseIcsContent(byte[] bytes) throws IOException { + BufferedReader reader = new BufferedReader(new StringReader(TestUtils.fromUtf8(bytes))); String line = reader.readLine(); if (!line.equals("BEGIN:VCALENDAR")) { throw new IllegalArgumentException();