#
# COPYRIGHT:
#
-# This software is Copyright (c) 1996-2012 Best Practical Solutions, LLC
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
# <sales@bestpractical.com>
#
# (Except where explicitly superseded by other copyright notices)
use Locale::Maketext::Lexicon 0.25;
use base 'Locale::Maketext::Fuzzy';
-use Encode;
use MIME::Entity;
use MIME::Head;
use File::Glob;
# do the same for parts first of all
SetMIMEEntityToEncoding( $_, $enc, $preserve_words ) foreach $entity->parts;
- my $charset = _FindOrGuessCharset($entity) or return;
+ my $head = $entity->head;
+
+ my $charset = _FindOrGuessCharset($entity);
+ if ( $charset ) {
+ unless( Encode::find_encoding($charset) ) {
+ $RT::Logger->warning("Encoding '$charset' is not supported");
+ $charset = undef;
+ }
+ }
+ unless ( $charset ) {
+ $head->replace( "X-RT-Original-Content-Type" => $head->mime_attr('Content-Type') );
+ $head->mime_attr('Content-Type' => 'application/octet-stream');
+ return;
+ }
SetMIMEHeadToEncoding(
- $entity->head,
+ $head,
_FindOrGuessCharset($entity, 1) => $enc,
$preserve_words
);
- my $head = $entity->head;
-
- # convert at least MIME word encoded attachment filename
- foreach my $attr (qw(content-type.name content-disposition.filename)) {
- if ( my $name = $head->mime_attr($attr) and !$preserve_words ) {
- $head->mime_attr( $attr => DecodeMIMEWordsToUTF8($name) );
- }
- }
-
# If this is a textual entity, we'd need to preserve its original encoding
- $head->replace( "X-RT-Original-Encoding" => $charset )
+ $head->replace( "X-RT-Original-Encoding" => Encode::encode( "UTF-8", $charset ) )
if $head->mime_attr('content-type.charset') or IsTextualContentType($head->mime_type);
return unless IsTextualContentType($head->mime_type);
my $body = $entity->bodyhandle;
- if ( $enc ne $charset && $body ) {
+ if ( $body && ($enc ne $charset || $enc =~ /^utf-?8(?:-strict)?$/i) ) {
my $string = $body->as_string or return;
+ RT::Util::assert_bytes($string);
$RT::Logger->debug( "Converting '$charset' to '$enc' for "
. $head->mime_type . " - "
- . ( $head->get('subject') || 'Subjectless message' ) );
+ . ( Encode::decode("UTF-8",$head->get('subject')) || 'Subjectless message' ) );
- # NOTE:: see the comments at the end of the sub.
- Encode::_utf8_off($string);
- Encode::from_to( $string, $charset => $enc );
+ {
+ no warnings 'utf8';
+ $string = Encode::encode( $enc, Encode::decode( $charset, $string) );
+ }
my $new_body = MIME::Body::InCore->new($string);
}
}
-# NOTES: Why Encode::_utf8_off before Encode::from_to
-#
-# All the strings in RT are utf-8 now. Quotes from Encode POD:
-#
-# [$length =] from_to($octets, FROM_ENC, TO_ENC [, CHECK])
-# ... The data in $octets must be encoded as octets and not as
-# characters in Perl's internal format. ...
-#
-# Not turning off the UTF-8 flag in the string will prevent the string
-# from conversion.
-
-
-
=head2 DecodeMIMEWordsToUTF8 $raw
An utility method which mimics MIME::Words::decode_mimewords, but only
-limited functionality. This function returns an utf-8 string.
-
-It returns the decoded string, or the original string if it's not
-encoded. Since the subroutine converts specified string into utf-8
-charset, it should not alter a subject written in English.
-
-Why not use MIME::Words directly? Because it fails in RT when I
-tried. Maybe it's ok now.
+limited functionality. Despite its name, this function returns the
+bytes of the string, in UTF-8.
=cut
my $to_charset = _CanonicalizeCharset(shift);
my $field = shift || '';
- my @list = $str =~ m/(.*?)=\?([^?]+)\?([QqBb])\?([^?]+)\?=([^=]*)/gcs;
+ # handle filename*=ISO-8859-1''%74%E9%73%74%2E%74%78%74, parameter value
+ # continuations, and similar syntax from RFC 2231
+ if ($field =~ /^Content-(Type|Disposition)/i) {
+ # This concatenates continued parameters and normalizes encoded params
+ # to QB encoded-words which we handle below
+ $str = MIME::Field::ParamVal->parse($str)->stringify;
+ }
+
+ # Pre-parse by removing all whitespace between encoded words
+ my $encoded_word = qr/
+ =\? # =?
+ ([^?]+?) # charset
+ (?:\*[^?]+)? # optional '*language'
+ \? # ?
+ ([QqBb]) # encoding
+ \? # ?
+ ([^?]+) # encoded string
+ \?= # ?=
+ /x;
+ $str =~ s/($encoded_word)\s+(?=$encoded_word)/$1/g;
+
+ # Also merge quoted-printable sections together, in case multiple
+ # octets of a single encoded character were split between chunks.
+ # Though not valid according to RFC 2047, this has been seen in the
+ # wild.
+ 1 while $str =~ s/(=\?[^?]+\?[Qq]\?)([^?]+)\?=\1([^?]+)\?=/$1$2$3?=/i;
+
+ # XXX TODO: use decode('MIME-Header', ...) and Encode::Alias to replace our
+ # custom MIME word decoding and charset canonicalization. We can't do this
+ # until we parse before decode, instead of the other way around.
+ my @list = $str =~ m/(.*?) # prefix
+ $encoded_word
+ ([^=]*) # trailing
+ /xgcs;
if ( @list ) {
# add everything that hasn't matched to the end of the latest
}
# now we have got a decoded subject, try to convert into the encoding
- unless ( $charset eq $to_charset ) {
- Encode::from_to( $enc_str, $charset, $to_charset );
+ if ( $charset ne $to_charset || $charset =~ /^utf-?8(?:-strict)?$/i ) {
+ if ( Encode::find_encoding($charset) ) {
+ Encode::from_to( $enc_str, $charset, $to_charset );
+ } else {
+ $RT::Logger->warning("Charset '$charset' is not supported");
+ $enc_str =~ s/[^[:print:]]/\357\277\275/g;
+ Encode::from_to( $enc_str, 'UTF-8', $to_charset )
+ unless $to_charset eq 'utf-8';
+ }
}
# XXX TODO: RT doesn't currently do the right thing with mime-encoded headers
$enc_str = qq{"$enc_str"}
if $enc_str =~ /[,;]/
and $enc_str !~ /^".*"$/
+ and $prefix !~ /"$/ and $trailing !~ /^"/
and (!$field || $field =~ /^(?:To$|From$|B?Cc$|Content-)/i);
$str .= $prefix . $enc_str . $trailing;
}
}
-# handle filename*=ISO-8859-1''%74%E9%73%74%2E%74%78%74, see also rfc 2231
- @list = $str =~ m/(.*?\*=)([^']*?)'([^']*?)'(\S+)(.*?)(?=(?:\*=|$))/gcs;
- if (@list) {
- $str = '';
- while (@list) {
- my ( $prefix, $charset, $language, $enc_str, $trailing ) =
- splice @list, 0, 5;
- $prefix =~ s/\*=$/=/; # remove the *
- $charset = _CanonicalizeCharset($charset);
- $enc_str =~ s/%(\w{2})/chr hex $1/eg;
- unless ( $charset eq $to_charset ) {
- Encode::from_to( $enc_str, $charset, $to_charset );
- }
- $enc_str = qq{"$enc_str"}
- if $enc_str =~ /[,;]/
- and $enc_str !~ /^".*"$/
- and (!$field || $field =~ /^(?:To$|From$|B?Cc$|Content-)/i);
- $str .= $prefix . $enc_str . $trailing;
- }
- }
-
# We might have \n without trailing whitespace, which will result in
# invalid headers.
$str =~ s/\n//g;
return if $charset eq $enc and $preserve_words;
+ RT::Util::assert_bytes( $head->as_string );
foreach my $tag ( $head->tags ) {
next unless $tag; # seen in wild: headers with no name
my @values = $head->get_all($tag);
$head->delete($tag);
foreach my $value (@values) {
- if ( $charset ne $enc ) {
- Encode::_utf8_off($value);
- Encode::from_to( $value, $charset => $enc );
+ if ( $charset ne $enc || $enc =~ /^utf-?8(?:-strict)?$/i ) {
+ no warnings 'utf8';
+ $value = Encode::encode( $enc, Encode::decode( $charset, $value) );
}
$value = DecodeMIMEWordsToEncoding( $value, $enc, $tag )
unless $preserve_words;